Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/periodics_cdc_integration_mysql_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-31r6l-lmcbg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ceae2f5321fd8cbeef849558e410240a1a29c100" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-31r6l" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git rev-list --no-walk b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=9de90e8c-2bb8-4b00-a948-55a0b699e2f2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-31r6l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a0e5b959a55caf4285c3f0608dcbd47ed90740dac7e4244afe4990642ac521c1 NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh pingcap_tiflow_periodics_cdc_integration_mysql_test_231-31r6l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.23.3' GODEBUG='' GOTELEMETRY='local' GOTELEMETRYDIR='/home/jenkins/.config/go/telemetry' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build832211858=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-zr1fh bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 59456 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 5ad14c1fb749c3d3 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 1 IN A 20.205.243.166 github.com. 1 IN A 20.27.177.113 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 04 12:01:15 UTC 2025 ;; MSG SIZE rcvd: 103 [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-master) 192245760 bytes in 1.01 secs (191029563 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] echo computeBranchFromPR component: tiflow, prTargetBranch: master, prTitle: , trunkBranch: master [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/.git/ .git HEAD is now at 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) POST git-upload-pack (383 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master HEAD is now at 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) 4c631d5951854ac91765b729fe3849d8e8d5ed8a 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) 3cf60079e codec(ticdc): modify DDL type support about debezium protocol (#12113) 9ed3d76c5 consumer(ticdc): adjust consumer log level (#12092) ✅ Checkout master 🎉 /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow ✅ ~~~~~ All done. ~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-master already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/third_party_download [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + export TIDB_BRANCH=master + TIDB_BRANCH=master + export PD_BRANCH=master + PD_BRANCH=master + export TIKV_BRANCH=master + TIKV_BRANCH=master + export TIFLASH_BRANCH=master + TIFLASH_BRANCH=master + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Downloading binaries... >>> Downloading tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz 2025-04-04 20:01:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz [620275338/620275338] -> "tmp/tidb-server.tar.gz" [1] >>> Downloading pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz 2025-04-04 20:02:05 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz [228444265/228444265] -> "tmp/pd-server.tar.gz" [1] >>> Downloading tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz 2025-04-04 20:02:38 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz [817400808/817400808] -> "tmp/tikv-server.tar.gz" [1] >>> Downloading tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz 2025-04-04 20:03:07 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz [553235170/553235170] -> "tmp/tiflash.tar.gz" [1] >>> Downloading minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2025-04-04 20:03:13 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> Downloading etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2025-04-04 20:03:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> Downloading schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2025-04-04 20:03:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] >>> Downloading go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2025-04-04 20:03:29 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> Downloading jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2025-04-04 20:03:30 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] Download SUCCESS + ls -alh ./bin total 2.0G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 4 20:03 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 4 20:03 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 20:03 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 4 20:03 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + make check_third_party_binary go: downloading go1.23.4 (linux/amd64) /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-region-bench ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 2.0G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 4 20:03 . drwxr-sr-x 3 jenkins jenkins 4.0K Apr 4 20:03 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 20:03 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 4 20:03 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + ./bin/tidb-server -V Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 + ./bin/tikv-server -V TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K Apr 4 20:03 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 4 20:03 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:03:36" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading github.com/spf13/pflag v1.0.5 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/xdg/scram v1.0.5 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/fatih/color v1.18.0 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/r3labs/diff v1.1.0 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/golang/mock v1.6.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/go-mysql-org/go-mysql v1.9.1 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading k8s.io/apimachinery v0.29.11 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:06:41" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer go: downloading github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 go: downloading github.com/DataDog/zstd v1.5.5 + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:22" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/sync_diff_inspector ./sync_diff_inspector go: downloading github.com/olekukonko/tablewriter v0.0.5 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20241224111350-ad2e4f976c7c go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading gorm.io/driver/mysql v1.5.7 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 12:09:30" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.4G drwxr-sr-x 2 jenkins jenkins 4.0K Apr 4 20:15 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 4 20:03 .. -rwxr-xr-x 1 jenkins jenkins 231M Apr 4 20:15 cdc -rwxr-xr-x 1 jenkins jenkins 363M Apr 4 20:15 cdc.test -rwxr-xr-x 1 jenkins jenkins 268M Apr 4 20:10 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 4 20:10 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 199M Apr 4 20:09 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 13M Apr 4 20:10 oauth2-server -rwxr-xr-x 1 jenkins jenkins 145M Apr 4 20:10 sync_diff_inspector + ./bin/cdc version Release Version: v9.0.0-beta.1-9-g4c631d595 Git Commit Hash: 4c631d5951854ac91765b729fe3849d8e8d5ed8a Git Branch: HEAD UTC Build Time: 2025-04-04 12:09:30 Go Version: go1.23.4 Failpoint Build: true [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-region-bench ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.3G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 4 20:15 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 4 20:03 .. drwxr-sr-x 2 jenkins jenkins 4.0K Apr 4 20:15 bin -rwxr-xr-x 1 jenkins jenkins 231M Apr 4 20:15 cdc -rwxr-xr-x 1 jenkins jenkins 363M Apr 4 20:15 cdc.test -rwxr-xr-x 1 jenkins jenkins 268M Apr 4 20:10 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 4 20:10 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 199M Apr 4 20:09 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K Apr 4 20:15 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 4 20:15 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 20:15 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 4 20:15 jq drwxr-sr-x 3 jenkins jenkins 4.0K Apr 4 20:15 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 4 20:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Apr 4 20:15 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 4 20:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Apr 4 20:15 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 4 20:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 4 20:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 4 20:15 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 4 20:15 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Apr 4 20:15 minio -rwxr-xr-x 1 jenkins jenkins 13M Apr 4 20:10 oauth2-server -rwxr-xr-x 1 jenkins jenkins 41M Apr 4 20:15 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 4 20:15 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 4 20:15 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 4 20:15 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 4 20:15 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 4 20:15 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 4 20:15 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 4 20:15 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 4 20:15 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K Apr 4 20:15 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 4 20:15 stores-dump -rwxr-xr-x 1 jenkins jenkins 145M Apr 4 20:10 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 241M Apr 4 20:15 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 4 20:15 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 4 20:15 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 4 20:15 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 53.54 secs (75248787 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-802hp-43s61 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ba495b72880ff51accbe2eb6764d769d2a776db2" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-802hp" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3e50d6a2; decorates RemoteLauncher[hudson.remoting.Channel@5112f19:JNLP4-connect connection from 10.233.90.156/10.233.90.156:36952] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rg35w-8m0fs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "581cc9cb131b045169b59497b65de8041953702c" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rg35w" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-96hff-bsj8l --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "58f2335f991763feff074f6c9ff4b68ccca0069a" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-96hff" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] checkout Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { The recommended git tool is: git Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-sszmz-b2zgt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "78d3baa32aef27de246324c453ccd31c02bf0497" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-sszmz" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] checkout Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-lsm90-sh1bf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bbaac64d09af601d9169a23bd0089bba2bc7034b" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-lsm90" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb [Pipeline] checkout The recommended git tool is: git Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-0n1jg-8b22m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f957a6ad283f35a53ee21a30f39629d0b5330016" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-0n1jg" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5509f364; decorates RemoteLauncher[hudson.remoting.Channel@5c17ad04:JNLP4-connect connection from 10.233.97.242/10.233.97.242:36910] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3zd69-tj81d --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a1a4614296442bf340017f8a346bc1ed14761668" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3zd69" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3a1a8ace; decorates RemoteLauncher[hudson.remoting.Channel@66bab76c:JNLP4-connect connection from 10.233.127.100/10.233.127.100:46198] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] checkout No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5757622; decorates RemoteLauncher[hudson.remoting.Channel@48bf3b1a:JNLP4-connect connection from 10.233.105.122/10.233.105.122:43220] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] node Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6d54cc71; decorates RemoteLauncher[hudson.remoting.Channel@5e0b1c33:JNLP4-connect connection from 10.233.86.224/10.233.86.224:45452] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@73dc80a2; decorates RemoteLauncher[hudson.remoting.Channel@75aa5f04:JNLP4-connect connection from 10.233.97.248/10.233.97.248:54334] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@388ea0c0; decorates RemoteLauncher[hudson.remoting.Channel@e3de7b0:JNLP4-connect connection from 10.233.72.32/10.233.72.32:52298] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-1lh2x-6xkw5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2fee64095f03e830adf26f35dbe3ba2ab8a20db2" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-1lh2x" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] podTemplate Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-8lmx3-ztb82 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "81eb33e5553b137461e6aaffc0772c8bddbb6d1d" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-8lmx3" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-hnsgf-qxgrr --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "74b8ede0226baea434128e1e20f73954b88621f4" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-hnsgf" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-kvssr-0xrf9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f292d8f3df80bfd6b7c8a44dfbb8d3bde8e4f1ad" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-kvssr" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-h5sch-9nqh9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "78a9ea0619237e6c90a6b2912ddfe520f5b25f2b" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-h5sch" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-zp6fw-wj7ls --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9ff8e1c6c8b7d9e2e1a3e0c6432431aafd51a451" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-zp6fw" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 7.41 secs (543859845 bytes/sec) [Pipeline] { [Pipeline] node [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth debezium_basic kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=38480815-9a5b-44cc-baa6-dc6f67948972 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-802hp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l pingcap_tiflow_periodics_cdc_integration_mysql_test_231-802hp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] container The recommended git tool is: git [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] { [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71 [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@41c17f97; decorates RemoteLauncher[hudson.remoting.Channel@40293dc:JNLP4-connect connection from 10.233.127.61/10.233.127.61:33480] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3e035caf; decorates RemoteLauncher[hudson.remoting.Channel@1a16af41:JNLP4-connect connection from 10.233.105.232/10.233.105.232:34328] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@76477d5c; decorates RemoteLauncher[hudson.remoting.Channel@69298097:JNLP4-connect connection from 10.233.90.247/10.233.90.247:54998] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4a6dab9b; decorates RemoteLauncher[hudson.remoting.Channel@428ae3ce:JNLP4-connect connection from 10.233.86.72/10.233.86.72:51724] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@25c0c707; decorates RemoteLauncher[hudson.remoting.Channel@2237657c:JNLP4-connect connection from 10.233.123.36/10.233.123.36:40358] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@60e7564e; decorates RemoteLauncher[hudson.remoting.Channel@7aaf589e:JNLP4-connect connection from 10.233.72.36/10.233.72.36:36984] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl (144kB) Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl (166kB) Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Downloading https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl (70kB) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3hjxs-08h21 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "96cb19e5cb0b17e0676dd5c9f9d72de22fc941b3" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3hjxs" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-32g4t-zdm9f --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4ca8234766a6fd8c6ee14d6c4a6dd525fc28e8d8" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-32g4t" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Installing collected packages: urllib3, certifi, idna, requests Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2025.1.31 idna-3.10 requests-2.26.0 urllib3-1.26.20 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-qshbg-3tnx7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7e6f0a20f824ea06f1865b40dbb123f45f73ee52" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-qshbg" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-j507b-l3t77 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cc91472d77f69b98aa93a08e5d28efcfed54dfa7" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-j507b" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test The 1 times to try to start tidb cluster... > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-t65nr-jkzwz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1bda7f47f631e01902b032b7ba41b617244c7ea5" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-t65nr" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-l1r54-82nmq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c7608e69f8fe51d32bc22a44335bafb2a0befd01" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-l1r54" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 7.03 secs (573286351 bytes/sec) [Pipeline] { [Pipeline] cache Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-wk88r-r525x --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e2e6888fd21e6d494dbef9e0e683514985668752" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-wk88r" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rc28z-rnw2c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b7c147aac340760fc7970833985781fcecd1e24e" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rc28z" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_231-7gw7k-w2p51 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cfa1fef9223f11a723a4f1495c18c27de429380e" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_231-7gw7k" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489430912868511324 TiDB Cluster ID. tikv_gc_leader_uuid 65802eeee50000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:1367, start at 2025-04-04 20:17:05.44943517 +0800 CST m=+2.304852058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:05.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:05.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:05.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489430912868511324 TiDB Cluster ID. tikv_gc_leader_uuid 65802eeee50000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:1367, start at 2025-04-04 20:17:05.44943517 +0800 CST m=+2.304852058 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:05.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:05.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:05.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489430910182504482 TiDB Cluster ID. tikv_gc_leader_uuid 65802eeeee4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:1443, start at 2025-04-04 20:17:05.481574068 +0800 CST m=+2.245349090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:05.534 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:05.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:05.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 7.88 secs (511181743 bytes/sec) [Pipeline] { [Pipeline] cache [Fri Apr 4 20:17:09 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27662768.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:17:12 GMT < Content-Length: 864 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b486d09e-d342-4f7f-872e-5767b4fa46e8 {"id":"b486d09e-d342-4f7f-872e-5767b4fa46e8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba2844c b486d09e-d342-4f7f-872e-5767b4fa46e8 /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b486d09e-d342-4f7f-872e-5767b4fa46e8 {"id":"b486d09e-d342-4f7f-872e-5767b4fa46e8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba2844c b486d09e-d342-4f7f-872e-5767b4fa46e8 /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b486d09e-d342-4f7f-872e-5767b4fa46e8 {"id":"b486d09e-d342-4f7f-872e-5767b4fa46e8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba2844c b486d09e-d342-4f7f-872e-5767b4fa46e8 /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 2771 owner id b486d09e-d342-4f7f-872e-5767b4fa46e8 default [Fri Apr 4 20:17:15 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.28992901.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 6.63 secs (607691390 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:17:18 GMT < Content-Length: 1325 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba {"id":"2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769035} /tidb/cdc/default/__cdc_meta__/capture/b486d09e-d342-4f7f-872e-5767b4fa46e8 {"id":"b486d09e-d342-4f7f-872e-5767b4fa46e8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba2844c b486d09e-d342-4f7f-872e-5767b4fa46e8 /tidb/cdc/default/__cdc_meta__/owner/22319600bba28489 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba {"id":"2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769035} /tidb/cdc/default/__cdc_meta__/capture/b486d09e-d342-4f7f-872e-5767b4fa46e8 {"id":"b486d09e-d342-4f7f-872e-5767b4fa46e8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba2844c b486d09e-d342-4f7f-872e-5767b4fa46e8 /tidb/cdc/default/__cdc_meta__/owner/22319600bba28489 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba {"id":"2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769035} /tidb/cdc/default/__cdc_meta__/capture/b486d09e-d342-4f7f-872e-5767b4fa46e8 {"id":"b486d09e-d342-4f7f-872e-5767b4fa46e8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba2844c b486d09e-d342-4f7f-872e-5767b4fa46e8 /tidb/cdc/default/__cdc_meta__/owner/22319600bba28489 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "b486d09e-d342-4f7f-872e-5767b4fa46e8 default" | grep id "id": "2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba", run task successfully capture_id: 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba pass test: check health pass test: get status pass test: create changefeed table test.simple1 not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 6.98 secs (577066954 bytes/sec) [Pipeline] { [Pipeline] cache table test.simple1 not exists for 2-th check, retry later table test.simple1 exists pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: update changefeed wait process 2771 exit for 1-th time... wait process 2771 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2771) - No such process wait process 2771 exit for 3-th time... process 2771 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba -A1 | grep '"is-owner": true' "is-owner": true, run task successfully [Fri Apr 4 20:17:34 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.31673169.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.21 secs (490984939 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:17:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7489430912868511324 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:17:21.902987495 +0800 CST StartTs:457118591699386376 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039eb050 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118591712231431} {CheckpointTs:457118591895994391 MinTableBarrierTs:457118595749511170 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing changefeedID: default/changefeed-test3 {UpstreamID:7489430912868511324 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2025-04-04 20:17:22.253503023 +0800 CST StartTs:457118591777767434 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0047c4870 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118591804243976} {CheckpointTs:457118591895994391 MinTableBarrierTs:457118595749511170 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba {"id":"2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769035} /tidb/cdc/default/__cdc_meta__/capture/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b {"id":"3915d6fd-3e7e-461f-ad9f-46d1d5959d7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769054} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba28489 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba /tidb/cdc/default/__cdc_meta__/owner/22319600bba28535 3915d6fd-3e7e-461f-ad9f-46d1d5959d7b /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:17:21.902987495+08:00","start-ts":457118591699386376,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118591712231431} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2025-04-04T20:17:22.10285541+08:00","start-ts":457118591738707976,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118593783431176} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2025-04-04T20:17:22.253503023+08:00","start-ts":457118591777767434,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118591804243976} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118595749511170,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118593638989835,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118595749511170,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7489430912868511324 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:17:21.902987495 +0800 CST StartTs:457118591699386376 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039eb050 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118591712231431} {CheckpointTs:457118591895994391 MinTableBarrierTs:457118595749511170 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing changefeedID: default/changefeed-test3 {UpstreamID:7489430912868511324 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2025-04-04 20:17:22.253503023 +0800 CST StartTs:457118591777767434 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0047c4870 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118591804243976} {CheckpointTs:457118591895994391 MinTableBarrierTs:457118595749511170 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba {"id":"2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769035} /tidb/cdc/default/__cdc_meta__/capture/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b {"id":"3915d6fd-3e7e-461f-ad9f-46d1d5959d7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769054} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba28489 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba /tidb/cdc/default/__cdc_meta__/owner/22319600bba28535 3915d6fd-3e7e-461f-ad9f-46d1d5959d7b /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:17:21.902987495+08:00","start-ts":457118591699386376,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118591712231431} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2025-04-04T20:17:22.10285541+08:00","start-ts":457118591738707976,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118593783431176} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2025-04-04T20:17:22.253503023+08:00","start-ts":457118591777767434,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118591804243976} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118595749511170,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118593638989835,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118595749511170,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7489430912868511324 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:17:21.902987495 +0800 CST StartTs:457118591699386376 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039eb050 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118591712231431} {CheckpointTs:457118591895994391 MinTableBarrierTs:457118595749511170 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing changefeedID: default/changefeed-test3 {UpstreamID:7489430912868511324 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2025-04-04 20:17:22.253503023 +0800 CST StartTs:457118591777767434 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0047c4870 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118591804243976} {CheckpointTs:457118591895994391 MinTableBarrierTs:457118595749511170 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118591895994391, checkpointTs: 457118591895994391, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba {"id":"2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769035} /tidb/cdc/default/__cdc_meta__/capture/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b {"id":"3915d6fd-3e7e-461f-ad9f-46d1d5959d7b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769054} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bba28489 2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba /tidb/cdc/default/__cdc_meta__/owner/22319600bba28535 3915d6fd-3e7e-461f-ad9f-46d1d5959d7b /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:17:21.902987495+08:00","start-ts":457118591699386376,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118591712231431} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2025-04-04T20:17:22.10285541+08:00","start-ts":457118591738707976,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118593783431176} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7489430912868511324,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2025-04-04T20:17:22.253503023+08:00","start-ts":457118591777767434,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118591804243976} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118595749511170,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118593638989835,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457118591895994391,"min-table-barrier-ts":457118595749511170,"admin-job-type":0} /tidb/cdc/default/default/task/position/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/2b0d46ba-0d21-4faa-bb2f-f6e4a8c0feba/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/3915d6fd-3e7e-461f-ad9f-46d1d5959d7b/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489430912868511324 {"id":7489430912868511324,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' "address": "127.0.0.1:8300", run task successfully pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: resume changefeed pass test: rebalance table pass test: move table pass test: get processors pass test: list processors pass test: set log level pass test: remove changefeed pass test: resign owner Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 7.63 secs (528037369 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=eaa22dcf-2457-4781-ae07-0637dc9ea6fa BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rg35w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491 pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rg35w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 [Pipeline] sh Run cases: row_format drop_many_tables processor_stop_delay partition_table ddl_with_exists PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=10979cfe-a1a9-4deb-b132-95a6ff3b5e7a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-96hff GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg pingcap_tiflow_periodics_cdc_integration_mysql_test_231-96hff GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=e6dd9f3f-b48e-4045-b391-3fe911a94452 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-sszmz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm pingcap_tiflow_periodics_cdc_integration_mysql_test_231-sszmz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=c21579e4-ca3c-483d-b6ec-0cb883426852 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-lsm90 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv pingcap_tiflow_periodics_cdc_integration_mysql_test_231-lsm90 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 Run cases: charset_gbk ddl_manager multi_source vector PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=dfc904fa-233d-4c7b-aaad-1f8a9399616d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-0n1jg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352 pingcap_tiflow_periodics_cdc_integration_mysql_test_231-0n1jg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=1aee03eb-ddfc-45f9-aeb1-2a074633170c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3zd69 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3zd69 pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] { (Test) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@262dd3f9; decorates RemoteLauncher[hudson.remoting.Channel@2bc0617f:JNLP4-connect connection from 10.233.90.94/10.233.90.94:41944] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4664fd5b; decorates RemoteLauncher[hudson.remoting.Channel@5da78eb8:JNLP4-connect connection from 10.233.86.125/10.233.86.125:48418] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@15e50343; decorates RemoteLauncher[hudson.remoting.Channel@429c1813:JNLP4-connect connection from 10.233.97.127/10.233.97.127:53750] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@299dd3e; decorates RemoteLauncher[hudson.remoting.Channel@6d8c7e35:JNLP4-connect connection from 10.233.105.241/10.233.105.241:41828] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32eb4307; decorates RemoteLauncher[hudson.remoting.Channel@eee4e54:JNLP4-connect connection from 10.233.127.149/10.233.127.149:59402] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@8cb1976; decorates RemoteLauncher[hudson.remoting.Channel@5aca9c42:JNLP4-connect connection from 10.233.126.96/10.233.126.96:47928] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@f5be826; decorates RemoteLauncher[hudson.remoting.Channel@1a4ad0cd:JNLP4-connect connection from 10.233.123.53/10.233.123.53:44070] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@604ea894; decorates RemoteLauncher[hudson.remoting.Channel@12858f72:JNLP4-connect connection from 10.233.72.104/10.233.72.104:38230] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5da888bf; decorates RemoteLauncher[hudson.remoting.Channel@4aea6a6d:JNLP4-connect connection from 10.233.73.41/10.233.73.41:51980] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Verifying downstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431139821791473 TiDB Cluster ID. tikv_gc_leader_uuid 65802f229b0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:1278, start at 2025-04-04 20:17:58.411792598 +0800 CST m=+2.576306104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431139821791473 TiDB Cluster ID. tikv_gc_leader_uuid 65802f229b0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:1278, start at 2025-04-04 20:17:58.411792598 +0800 CST m=+2.576306104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.420 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431139633634011 TiDB Cluster ID. tikv_gc_leader_uuid 65802f22b54000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:1351, start at 2025-04-04 20:17:58.521602793 +0800 CST m=+2.605745711 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.530 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.535 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.535 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431143577828966 TiDB Cluster ID. tikv_gc_leader_uuid 65802f22f540005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:1335, start at 2025-04-04 20:17:58.747619118 +0800 CST m=+2.341407226 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431143577828966 TiDB Cluster ID. tikv_gc_leader_uuid 65802f22f540005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:1335, start at 2025-04-04 20:17:58.747619118 +0800 CST m=+2.341407226 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431139277779664 TiDB Cluster ID. tikv_gc_leader_uuid 65802f22f40000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:1394, start at 2025-04-04 20:17:58.768741559 +0800 CST m=+2.290909310 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.776 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.736 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.736 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431142639106615 TiDB Cluster ID. tikv_gc_leader_uuid 65802f233cc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:1279, start at 2025-04-04 20:17:59.039234338 +0800 CST m=+2.017850762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431142639106615 TiDB Cluster ID. tikv_gc_leader_uuid 65802f233cc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:1279, start at 2025-04-04 20:17:59.039234338 +0800 CST m=+2.017850762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431133989653614 TiDB Cluster ID. tikv_gc_leader_uuid 65802f2355c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:1354, start at 2025-04-04 20:17:59.146023873 +0800 CST m=+2.041566527 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.153 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431136793147100 TiDB Cluster ID. tikv_gc_leader_uuid 65802f23a600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:1218, start at 2025-04-04 20:17:59.491774773 +0800 CST m=+2.104684928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431136793147100 TiDB Cluster ID. tikv_gc_leader_uuid 65802f23a600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:1218, start at 2025-04-04 20:17:59.491774773 +0800 CST m=+2.104684928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431146331474687 TiDB Cluster ID. tikv_gc_leader_uuid 65802f23d7c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:1285, start at 2025-04-04 20:17:59.649649948 +0800 CST m=+2.175969683 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.658 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.647 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.647 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.73 secs (461625306 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431146864656921 TiDB Cluster ID. tikv_gc_leader_uuid 65802f24188000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:1209, start at 2025-04-04 20:17:59.927139273 +0800 CST m=+2.214558541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431146864656921 TiDB Cluster ID. tikv_gc_leader_uuid 65802f24188000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:1209, start at 2025-04-04 20:17:59.927139273 +0800 CST m=+2.214558541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:59.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:59.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:59.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431142133849298 TiDB Cluster ID. tikv_gc_leader_uuid 65802f244a40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:1291, start at 2025-04-04 20:18:00.117562637 +0800 CST m=+2.304883848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:00.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:00.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:00.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 4 20:18:02 CST 2025] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26942696.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2633.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2670.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 4 20:18:03 CST 2025] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.26732675.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 20:18:04 CST 2025] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26952697.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457118602517544962 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118602517544962 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:18:04 CST 2025] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26772679.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457118602671423489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118602671423489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readallfields"="true" "threadcount"="2" "operationcount"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "dotransactions"="false" "readproportion"="0" "mysql.user"="root" "recordcount"="10" "requestdistribution"="uniform" "updateproportion"="0" "mysql.db"="sink_retry" "scanproportion"="0" ********************************************** Run finished, takes 7.408376ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2208.0, Avg(us): 1400, Min(us): 869, Max(us): 2767, 95th(us): 3000, 99th(us): 3000 [Fri Apr 4 20:18:05 CST 2025] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.27252727.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:05 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67e49065-b249-4a51-b627-d4a9e0bd6c56 {"id":"67e49065-b249-4a51-b627-d4a9e0bd6c56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc706cd5 67e49065-b249-4a51-b627-d4a9e0bd6c56 /tidb/cdc/default/default/upstream/7489431139821791473 {"id":7489431139821791473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67e49065-b249-4a51-b627-d4a9e0bd6c56 {"id":"67e49065-b249-4a51-b627-d4a9e0bd6c56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc706cd5 67e49065-b249-4a51-b627-d4a9e0bd6c56 /tidb/cdc/default/default/upstream/7489431139821791473 {"id":7489431139821791473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67e49065-b249-4a51-b627-d4a9e0bd6c56 {"id":"67e49065-b249-4a51-b627-d4a9e0bd6c56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc706cd5 67e49065-b249-4a51-b627-d4a9e0bd6c56 /tidb/cdc/default/default/upstream/7489431139821791473 {"id":7489431139821791473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 49f64364-9b5d-4bbd-b700-1d7db4c94a7b Info: {"upstream_id":7489431139821791473,"namespace":"default","id":"49f64364-9b5d-4bbd-b700-1d7db4c94a7b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:18:05.590885363+08:00","start_ts":457118602318315521,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118602318315521,"checkpoint_ts":457118602318315521,"checkpoint_time":"2025-04-04 20:18:02.330"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:06 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0450ab21-fb5c-4cfa-bfd9-12002ba2fea7 {"id":"0450ab21-fb5c-4cfa-bfd9-12002ba2fea7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6e8bdb 0450ab21-fb5c-4cfa-bfd9-12002ba2fea7 /tidb/cdc/default/default/upstream/7489431136793147100 {"id":7489431136793147100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0450ab21-fb5c-4cfa-bfd9-12002ba2fea7 {"id":"0450ab21-fb5c-4cfa-bfd9-12002ba2fea7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6e8bdb 0450ab21-fb5c-4cfa-bfd9-12002ba2fea7 /tidb/cdc/default/default/upstream/7489431136793147100 {"id":7489431136793147100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0450ab21-fb5c-4cfa-bfd9-12002ba2fea7 {"id":"0450ab21-fb5c-4cfa-bfd9-12002ba2fea7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6e8bdb 0450ab21-fb5c-4cfa-bfd9-12002ba2fea7 /tidb/cdc/default/default/upstream/7489431136793147100 {"id":7489431136793147100,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "recordcount"="50" "updateproportion"="0" "workload"="core" "requestdistribution"="uniform" "mysql.db"="consistent_replicate_ddl" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" "mysql.user"="root" "insertproportion"="0" "dotransactions"="false" "threadcount"="10" "mysql.host"="127.0.0.1" "readallfields"="true" "fieldcount"="3" "scanproportion"="0" ********************************************** Run finished, takes 11.119162ms INSERT - Takes(s): 0.0, Count: 50, OPS: 7805.1, Avg(us): 2020, Min(us): 1056, Max(us): 5029, 95th(us): 5000, 99th(us): 6000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:07 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d {"id":"9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc71e5d5 9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d /tidb/cdc/default/default/upstream/7489431146864656921 {"id":7489431146864656921,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d {"id":"9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc71e5d5 9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d /tidb/cdc/default/default/upstream/7489431146864656921 {"id":7489431146864656921,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d {"id":"9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc71e5d5 9f8f34a2-2b4f-4b6f-950f-7ce93a85eb7d /tidb/cdc/default/default/upstream/7489431146864656921 {"id":7489431146864656921,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:07 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83d363fc-c33c-467f-a397-5ac5f05a2775 {"id":"83d363fc-c33c-467f-a397-5ac5f05a2775","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6cbee6 83d363fc-c33c-467f-a397-5ac5f05a2775 /tidb/cdc/default/default/upstream/7489431143577828966 {"id":7489431143577828966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83d363fc-c33c-467f-a397-5ac5f05a2775 {"id":"83d363fc-c33c-467f-a397-5ac5f05a2775","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6cbee6 83d363fc-c33c-467f-a397-5ac5f05a2775 /tidb/cdc/default/default/upstream/7489431143577828966 {"id":7489431143577828966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83d363fc-c33c-467f-a397-5ac5f05a2775 {"id":"83d363fc-c33c-467f-a397-5ac5f05a2775","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769084} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6cbee6 83d363fc-c33c-467f-a397-5ac5f05a2775 /tidb/cdc/default/default/upstream/7489431143577828966 {"id":7489431143577828966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2732.out cli changefeed create --start-ts=457118602517544962 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: ad79464a-f8db-4849-a386-f5f73977271d Info: {"upstream_id":7489431143577828966,"namespace":"default","id":"ad79464a-f8db-4849-a386-f5f73977271d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:18:08.00559933+08:00","start_ts":457118602517544962,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118602517544962,"checkpoint_ts":457118602517544962,"checkpoint_time":"2025-04-04 20:18:03.090"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:08 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/163e5dd4-4c02-46c2-879f-aa33ee76f54c {"id":"163e5dd4-4c02-46c2-879f-aa33ee76f54c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6f3df4 163e5dd4-4c02-46c2-879f-aa33ee76f54c /tidb/cdc/default/default/upstream/7489431142639106615 {"id":7489431142639106615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/163e5dd4-4c02-46c2-879f-aa33ee76f54c {"id":"163e5dd4-4c02-46c2-879f-aa33ee76f54c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6f3df4 163e5dd4-4c02-46c2-879f-aa33ee76f54c /tidb/cdc/default/default/upstream/7489431142639106615 {"id":7489431142639106615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/163e5dd4-4c02-46c2-879f-aa33ee76f54c {"id":"163e5dd4-4c02-46c2-879f-aa33ee76f54c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6f3df4 163e5dd4-4c02-46c2-879f-aa33ee76f54c /tidb/cdc/default/default/upstream/7489431142639106615 {"id":7489431142639106615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2787.out cli changefeed create --start-ts=457118602671423489 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 5e1b9017-e502-4f18-bcf0-59f6213dc0ec Info: {"upstream_id":7489431142639106615,"namespace":"default","id":"5e1b9017-e502-4f18-bcf0-59f6213dc0ec","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:18:08.671971821+08:00","start_ts":457118602671423489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118602671423489,"checkpoint_ts":457118602671423489,"checkpoint_time":"2025-04-04 20:18:03.677"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 6.88 secs (585746586 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 not exists for 3-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table charset_gbk_test1.t0 not exists for 2-th check, retry later table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later table test.finish_mark not exists for 2-th check, retry later table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: get tso wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:18:18 CST 2025] <<<<<< run test case http_api success! >>>>>> table test.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 7.07 secs (569836498 bytes/sec) [Pipeline] { [Pipeline] cache table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431140356573468 TiDB Cluster ID. tikv_gc_leader_uuid 65802f3874c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:1256, start at 2025-04-04 20:18:20.787779911 +0800 CST m=+24.034716757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:20.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:20.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:20.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later table test.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431140356573468 TiDB Cluster ID. tikv_gc_leader_uuid 65802f3874c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:1256, start at 2025-04-04 20:18:20.787779911 +0800 CST m=+24.034716757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:20.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:20.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:20.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431143204504658 TiDB Cluster ID. tikv_gc_leader_uuid 65802f22a38000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:1328, start at 2025-04-04 20:17:58.442944078 +0800 CST m=+1.618400788 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:19:58.450 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:17:58.414 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:07:58.414 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... table row_format.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:18:24 CST 2025] <<<<<< run test case charset_gbk success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2727.out cli tso query --pd=http://127.0.0.1:2379 table consistent_replicate_ddl.usertable_bak exists table row_format.finish_mark not exists for 2-th check, retry later + set +x + tso='457118608393502721 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118608393502721 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:18:26 CST 2025] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.27652767.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) table row_format.finish_mark not exists for 3-th check, retry later Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:30 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a7f8f919-4e04-4309-9b1e-a0b58d03b3c9 {"id":"a7f8f919-4e04-4309-9b1e-a0b58d03b3c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769107} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6c394c a7f8f919-4e04-4309-9b1e-a0b58d03b3c9 /tidb/cdc/default/default/upstream/7489431140356573468 {"id":7489431140356573468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a7f8f919-4e04-4309-9b1e-a0b58d03b3c9 {"id":"a7f8f919-4e04-4309-9b1e-a0b58d03b3c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769107} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6c394c a7f8f919-4e04-4309-9b1e-a0b58d03b3c9 /tidb/cdc/default/default/upstream/7489431140356573468 {"id":7489431140356573468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a7f8f919-4e04-4309-9b1e-a0b58d03b3c9 {"id":"a7f8f919-4e04-4309-9b1e-a0b58d03b3c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769107} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6c394c a7f8f919-4e04-4309-9b1e-a0b58d03b3c9 /tidb/cdc/default/default/upstream/7489431140356573468 {"id":7489431140356573468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2819.out cli changefeed create --start-ts=457118608393502721 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/conf/cf.toml Create changefeed successfully! ID: 174981d9-0515-40e8-b064-bdb001bc5cfc Info: {"upstream_id":7489431140356573468,"namespace":"default","id":"174981d9-0515-40e8-b064-bdb001bc5cfc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:18:30.391636062+08:00","start_ts":457118608393502721,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!foreign_key.t12"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118608393502721,"checkpoint_ts":457118608393502721,"checkpoint_time":"2025-04-04 20:18:25.505"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark not exists for 4-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 9.00 secs (447564722 bytes/sec) [Pipeline] { [Pipeline] cache + set +x wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:18:31 CST 2025] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.29142916.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table row_format.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {UpstreamID:7489431136793147100 Namespace:default ID:d6bea060-b84c-4e96-b94b-7a358dc9e0d1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:07.036941826 +0800 CST StartTs:457118603528896517 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039b2cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118603541741575} {CheckpointTs:457118609164992526 MinTableBarrierTs:457118610750701572 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86b6bc44-98a5-4ab5-877a-8ce2b1e769f3 {"id":"86b6bc44-98a5-4ab5-877a-8ce2b1e769f3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6e8d24 86b6bc44-98a5-4ab5-877a-8ce2b1e769f3 /tidb/cdc/default/default/changefeed/info/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"upstream-id":7489431136793147100,"namespace":"default","changefeed-id":"d6bea060-b84c-4e96-b94b-7a358dc9e0d1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:18:07.036941826+08:00","start-ts":457118603528896517,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118603541741575} /tidb/cdc/default/default/changefeed/status/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"checkpoint-ts":457118609164992526,"min-table-barrier-ts":457118610750701572,"admin-job-type":0} /tidb/cdc/default/default/task/position/86b6bc44-98a5-4ab5-877a-8ce2b1e769f3/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431136793147100 {"id":7489431136793147100,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {UpstreamID:7489431136793147100 Namespace:default ID:d6bea060-b84c-4e96-b94b-7a358dc9e0d1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:07.036941826 +0800 CST StartTs:457118603528896517 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039b2cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118603541741575} {CheckpointTs:457118609164992526 MinTableBarrierTs:457118610750701572 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86b6bc44-98a5-4ab5-877a-8ce2b1e769f3 {"id":"86b6bc44-98a5-4ab5-877a-8ce2b1e769f3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6e8d24 86b6bc44-98a5-4ab5-877a-8ce2b1e769f3 /tidb/cdc/default/default/changefeed/info/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"upstream-id":7489431136793147100,"namespace":"default","changefeed-id":"d6bea060-b84c-4e96-b94b-7a358dc9e0d1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:18:07.036941826+08:00","start-ts":457118603528896517,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118603541741575} /tidb/cdc/default/default/changefeed/status/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"checkpoint-ts":457118609164992526,"min-table-barrier-ts":457118610750701572,"admin-job-type":0} /tidb/cdc/default/default/task/position/86b6bc44-98a5-4ab5-877a-8ce2b1e769f3/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431136793147100 {"id":7489431136793147100,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {UpstreamID:7489431136793147100 Namespace:default ID:d6bea060-b84c-4e96-b94b-7a358dc9e0d1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:07.036941826 +0800 CST StartTs:457118603528896517 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039b2cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118603541741575} {CheckpointTs:457118609164992526 MinTableBarrierTs:457118610750701572 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457118609164992526, checkpointTs: 457118609164992526, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/86b6bc44-98a5-4ab5-877a-8ce2b1e769f3 {"id":"86b6bc44-98a5-4ab5-877a-8ce2b1e769f3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bc6e8d24 86b6bc44-98a5-4ab5-877a-8ce2b1e769f3 /tidb/cdc/default/default/changefeed/info/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"upstream-id":7489431136793147100,"namespace":"default","changefeed-id":"d6bea060-b84c-4e96-b94b-7a358dc9e0d1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:18:07.036941826+08:00","start-ts":457118603528896517,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118603541741575} /tidb/cdc/default/default/changefeed/status/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"checkpoint-ts":457118609164992526,"min-table-barrier-ts":457118610750701572,"admin-job-type":0} /tidb/cdc/default/default/task/position/86b6bc44-98a5-4ab5-877a-8ce2b1e769f3/d6bea060-b84c-4e96-b94b-7a358dc9e0d1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431136793147100 {"id":7489431136793147100,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark not exists for 7-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table row_format.finish_mark not exists for 8-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 6.66 secs (604902312 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:18:40 CST 2025] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table row_format.finish_mark exists check diff successfully table foreign_key.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431320380246055 TiDB Cluster ID. tikv_gc_leader_uuid 65802f4ab100035 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:4225, start at 2025-04-04 20:18:39.44887393 +0800 CST m=+2.130531727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:39.456 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:39.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:39.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431320380246055 TiDB Cluster ID. tikv_gc_leader_uuid 65802f4ab100035 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:4225, start at 2025-04-04 20:18:39.44887393 +0800 CST m=+2.130531727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:39.456 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:39.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:39.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431317584061633 TiDB Cluster ID. tikv_gc_leader_uuid 65802f4ad28000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:4306, start at 2025-04-04 20:18:39.590402337 +0800 CST m=+2.214306188 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:39.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:39.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:39.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "scanproportion"="0" "updateproportion"="0" "mysql.db"="sink_retry" "readallfields"="true" "workload"="core" "recordcount"="10" "dotransactions"="false" "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.port"="4000" "threadcount"="2" "insertproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 4.385795ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3685.9, Avg(us): 800, Min(us): 445, Max(us): 1695, 95th(us): 2000, 99th(us): 2000 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:18:42 CST 2025] <<<<<< run test case row_format success! >>>>>> table foreign_key.finish_mark not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 table foreign_key.finish_mark not exists for 7-th check, retry later Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 8-th check, retry later Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431342637464376 TiDB Cluster ID. tikv_gc_leader_uuid 65802f510d00008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:3919, start at 2025-04-04 20:18:45.973173222 +0800 CST m=+2.631020461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:45.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:45.989 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:45.989 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431342637464376 TiDB Cluster ID. tikv_gc_leader_uuid 65802f510d00008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:3919, start at 2025-04-04 20:18:45.973173222 +0800 CST m=+2.631020461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:45.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:45.989 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:45.989 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431343601038998 TiDB Cluster ID. tikv_gc_leader_uuid 65802f51174000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:3998, start at 2025-04-04 20:18:46.000149123 +0800 CST m=+2.597552754 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:46.009 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:45.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:45.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table foreign_key.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431360864699358 TiDB Cluster ID. tikv_gc_leader_uuid 65802f538380004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:5960, start at 2025-04-04 20:18:48.466799018 +0800 CST m=+1.410111897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:20:48.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:18:48.462 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:08:48.462 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Fri Apr 4 20:18:49 CST 2025] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.60056007.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 20:18:49 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52485250.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.47 secs (475838193 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh table foreign_key.finish_mark not exists for 10-th check, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=857567b2-9821-488e-b780-e80e5f26c921 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-1lh2x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-1lh2x pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=02a3099a-b53d-4297-9b74-363c8de45052 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-8lmx3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-8lmx3 pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=0fffe88b-3494-47a5-b3ff-df7eb251661b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-hnsgf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-hnsgf pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=2930db5c-e9b0-4ce6-8a4a-9a838e17810f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-kvssr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24 pingcap_tiflow_periodics_cdc_integration_mysql_test_231-kvssr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=386edacf-0173-4b5e-a3eb-63c024904002 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-h5sch GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-h5sch pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 table foreign_key.finish_mark not exists for 11-th check, retry later Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=3a19eb60-b65e-4d0b-9d10-901e778f5924 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-zp6fw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78 pingcap_tiflow_periodics_cdc_integration_mysql_test_231-zp6fw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:52 GMT < Content-Length: 1253 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/398db5bf-83c5-4177-ba11-879ba21a3d89 {"id":"398db5bf-83c5-4177-ba11-879ba21a3d89","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769129} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd7db 398db5bf-83c5-4177-ba11-879ba21a3d89 /tidb/cdc/default/default/upstream/7489431360864699358 {"id":7489431360864699358,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/398db5bf-83c5-4177-ba11-879ba21a3d89 {"id":"398db5bf-83c5-4177-ba11-879ba21a3d89","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769129} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd7db 398db5bf-83c5-4177-ba11-879ba21a3d89 /tidb/cdc/default/default/upstream/7489431360864699358 {"id":7489431360864699358,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/398db5bf-83c5-4177-ba11-879ba21a3d89 {"id":"398db5bf-83c5-4177-ba11-879ba21a3d89","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769129} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd7db 398db5bf-83c5-4177-ba11-879ba21a3d89 /tidb/cdc/default/default/upstream/7489431360864699358 {"id":7489431360864699358,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:52 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9666cd65-1a04-46c3-9958-4d433c9581bf {"id":"9666cd65-1a04-46c3-9958-4d433c9581bf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769130} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd2618d5 9666cd65-1a04-46c3-9958-4d433c9581bf /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9666cd65-1a04-46c3-9958-4d433c9581bf {"id":"9666cd65-1a04-46c3-9958-4d433c9581bf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769130} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd2618d5 9666cd65-1a04-46c3-9958-4d433c9581bf /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9666cd65-1a04-46c3-9958-4d433c9581bf {"id":"9666cd65-1a04-46c3-9958-4d433c9581bf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769130} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd2618d5 9666cd65-1a04-46c3-9958-4d433c9581bf /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5315.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7489431342637464376,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T20:18:53.252089887+08:00","start_ts":457118615637590019,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118615637590019,"checkpoint_ts":457118615637590019,"checkpoint_time":"2025-04-04 20:18:53.139"} PASS The 1 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Fri Apr 4 20:18:54 CST 2025] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.60726074.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x table foreign_key.finish_mark not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { wait process 5253 exit for 1-th time... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Verifying downstream PD is started... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { wait process 5253 exit for 2-th time... table foreign_key.finish_mark exists check diff successfully table foreign_key.t12 does not exists wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process 5253 exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) wait process cdc.test exit for 2-th time... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) wait process 5253 exit for 4-th time... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:18:57 GMT < Content-Length: 1714 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/398db5bf-83c5-4177-ba11-879ba21a3d89 {"id":"398db5bf-83c5-4177-ba11-879ba21a3d89","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769129} /tidb/cdc/default/__cdc_meta__/capture/58af001c-43c7-4ac0-b470-172c2bd0d39c {"id":"58af001c-43c7-4ac0-b470-172c2bd0d39c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769134} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd7db 398db5bf-83c5-4177-ba11-879ba21a3d89 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd819 58af001c-43c7-4ac0-b470-172c2bd0d39c /tidb/cdc/default/default/upstream/7489431360864699358 {"id":7489431360864699358,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/398db5bf-83c5-4177-ba11-879ba21a3d89 {"id":"398db5bf-83c5-4177-ba11-879ba21a3d89","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769129} /tidb/cdc/default/__cdc_meta__/capture/58af001c-43c7-4ac0-b470-172c2bd0d39c {"id":"58af001c-43c7-4ac0-b470-172c2bd0d39c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769134} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd7db 398db5bf-83c5-4177-ba11-879ba21a3d89 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd819 58af001c-43c7-4ac0-b470-172c2bd0d39c /tidb/cdc/default/default/upstream/7489431360864699358 {"id":7489431360864699358,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/398db5bf-83c5-4177-ba11-879ba21a3d89 {"id":"398db5bf-83c5-4177-ba11-879ba21a3d89","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769129} /tidb/cdc/default/__cdc_meta__/capture/58af001c-43c7-4ac0-b470-172c2bd0d39c {"id":"58af001c-43c7-4ac0-b470-172c2bd0d39c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769134} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd7db 398db5bf-83c5-4177-ba11-879ba21a3d89 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600bd3bd819 58af001c-43c7-4ac0-b470-172c2bd0d39c /tidb/cdc/default/default/upstream/7489431360864699358 {"id":7489431360864699358,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 5253 exit for 5-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:18:58 CST 2025] <<<<<< run test case foreign_key success! >>>>>> [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache wait process 5253 exit for 6-th time... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process 5253 exit for 7-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... pass test: check health pass test: get status wait process 5253 exit for 8-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 5253 exit for 9-th time... Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5253) - No such process wait process 5253 exit for 10-th time... process 5253 already exit + info='{ "upstream_id": 7489431360864699358, [Fri Apr 4 20:19:00 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' "namespace": "default", "id": "changefeed-test1", "state": "normal", + etcd_info_msg='etcd info' + '[' -z '' ']' "checkpoint_tso": 457118617439305739, "checkpoint_time": "2025-04-04 20:19:00.012", "error": null }' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + echo '{ "upstream_id": 7489431360864699358, "namespace": "default", + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457118617439305739, "checkpoint_time": "2025-04-04 20:19:00.012", "error": null }' { "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test1", + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53955397.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default "state": "normal", "checkpoint_tso": 457118617439305739, "checkpoint_time": "2025-04-04 20:19:00.012", ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... "error": null } * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457118617439305739, '"checkpoint_time":' '"2025-04-04' '20:19:00.012",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457118617439305739, '"checkpoint_time":' '"2025-04-04' '20:19:00.012",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457118617478889478, "checkpoint_time": "2025-04-04 20:19:00.163", "error": null }' + echo '{ "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457118617478889478, "checkpoint_time": "2025-04-04 20:19:00.163", "error": null }' { "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457118617478889478, "checkpoint_time": "2025-04-04 20:19:00.163", "error": null } ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457118617478889478, '"checkpoint_time":' '"2025-04-04' '20:19:00.163",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457118617478889478, '"checkpoint_time":' '"2025-04-04' '20:19:00.163",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + info='{ "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457118617517948933, "checkpoint_time": "2025-04-04 20:19:00.312", "error": null }' + echo '{ "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457118617517948933, "checkpoint_time": "2025-04-04 20:19:00.312", "error": null }' { "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457118617517948933, "checkpoint_time": "2025-04-04 20:19:00.312", "error": null } ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457118617517948933, '"checkpoint_time":' '"2025-04-04' '20:19:00.312",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457118617517948933, '"checkpoint_time":' '"2025-04-04' '20:19:00.312",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457118617557532681, "checkpoint_time": "2025-04-04 20:19:00.463", "error": null }' + echo '{ "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457118617557532681, "checkpoint_time": "2025-04-04 20:19:00.463", "error": null }' { "upstream_id": 7489431360864699358, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457118617557532681, "checkpoint_time": "2025-04-04 20:19:00.463", "error": null } ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457118617557532681, '"checkpoint_time":' '"2025-04-04' '20:19:00.463",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431360864699358, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457118617557532681, '"checkpoint_time":' '"2025-04-04' '20:19:00.463",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431417823088815 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61334000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:1274, start at 2025-04-04 20:19:02.493080643 +0800 CST m=+2.362721034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:02.541 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:02.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:02.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431417823088815 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61334000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:1274, start at 2025-04-04 20:19:02.493080643 +0800 CST m=+2.362721034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:02.541 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:02.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:02.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431412507752613 TiDB Cluster ID. tikv_gc_leader_uuid 65802f614c4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:1359, start at 2025-04-04 20:19:02.608712608 +0800 CST m=+2.407241270 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:02.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:02.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:02.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431409970276193 TiDB Cluster ID. tikv_gc_leader_uuid 65802f617680012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:1283, start at 2025-04-04 20:19:02.778527113 +0800 CST m=+2.643455638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:02.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:02.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:02.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431409970276193 TiDB Cluster ID. tikv_gc_leader_uuid 65802f617680012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:1283, start at 2025-04-04 20:19:02.778527113 +0800 CST m=+2.643455638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:02.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:02.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:02.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431416628371635 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61714000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:1357, start at 2025-04-04 20:19:02.742353414 +0800 CST m=+2.516566985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:02.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:02.744 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:02.744 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431418046017254 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61dd40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:1349, start at 2025-04-04 20:19:03.168667281 +0800 CST m=+2.832986001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.178 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431418046017254 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61dd40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:1349, start at 2025-04-04 20:19:03.168667281 +0800 CST m=+2.832986001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.178 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431412692422369 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61ce80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:1407, start at 2025-04-04 20:19:03.113183014 +0800 CST m=+2.711322021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.160 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489431342637464376 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:53.252089887 +0800 CST StartTs:457118615637590019 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035fb8c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118615663804419} {CheckpointTs:457118616044175385 MinTableBarrierTs:457118616044175385 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e83a1f5a-6162-4d94-b916-035528c21426 {"id":"e83a1f5a-6162-4d94-b916-035528c21426","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd261a93 e83a1f5a-6162-4d94-b916-035528c21426 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489431342637464376,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:18:53.252089887+08:00","start-ts":457118615637590019,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118615663804419} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457118616044175385,"min-table-barrier-ts":457118616044175385,"admin-job-type":0} /tidb/cdc/default/default/task/position/e83a1f5a-6162-4d94-b916-035528c21426/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489431342637464376 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:53.252089887 +0800 CST StartTs:457118615637590019 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035fb8c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118615663804419} {CheckpointTs:457118616044175385 MinTableBarrierTs:457118616044175385 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e83a1f5a-6162-4d94-b916-035528c21426 {"id":"e83a1f5a-6162-4d94-b916-035528c21426","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd261a93 e83a1f5a-6162-4d94-b916-035528c21426 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489431342637464376,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:18:53.252089887+08:00","start-ts":457118615637590019,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118615663804419} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457118616044175385,"min-table-barrier-ts":457118616044175385,"admin-job-type":0} /tidb/cdc/default/default/task/position/e83a1f5a-6162-4d94-b916-035528c21426/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489431342637464376 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:53.252089887 +0800 CST StartTs:457118615637590019 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035fb8c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118615663804419} {CheckpointTs:457118616044175385 MinTableBarrierTs:457118616044175385 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e83a1f5a-6162-4d94-b916-035528c21426 {"id":"e83a1f5a-6162-4d94-b916-035528c21426","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd261a93 e83a1f5a-6162-4d94-b916-035528c21426 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489431342637464376,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:18:53.252089887+08:00","start-ts":457118615637590019,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118615663804419} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457118616044175385,"min-table-barrier-ts":457118616044175385,"admin-job-type":0} /tidb/cdc/default/default/task/position/e83a1f5a-6162-4d94-b916-035528c21426/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:19:04 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54535455.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431413369416632 TiDB Cluster ID. tikv_gc_leader_uuid 65802f6222c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:3953, start at 2025-04-04 20:19:03.471835208 +0800 CST m=+2.595076634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.479 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431413369416632 TiDB Cluster ID. tikv_gc_leader_uuid 65802f6222c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:3953, start at 2025-04-04 20:19:03.471835208 +0800 CST m=+2.595076634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.479 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431415313013703 TiDB Cluster ID. tikv_gc_leader_uuid 65802f623b8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:4031, start at 2025-04-04 20:19:03.578843462 +0800 CST m=+2.629002194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.588 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431419166727409 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61f5c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:1222, start at 2025-04-04 20:19:03.272277246 +0800 CST m=+2.309666157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.255 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.255 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431419166727409 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61f5c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:1222, start at 2025-04-04 20:19:03.272277246 +0800 CST m=+2.309666157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.255 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.255 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431415874629628 TiDB Cluster ID. tikv_gc_leader_uuid 65802f61e90000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:1299, start at 2025-04-04 20:19:03.253592829 +0800 CST m=+2.200856037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:03.261 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:03.254 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:03.254 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.verify_table_eligible not exists for 1-th check, retry later [Fri Apr 4 20:19:06 CST 2025] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26592661.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431425782587193 TiDB Cluster ID. tikv_gc_leader_uuid 65802f631dc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:4255, start at 2025-04-04 20:19:04.465509108 +0800 CST m=+2.019269741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:04.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:04.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:04.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431425782587193 TiDB Cluster ID. tikv_gc_leader_uuid 65802f631dc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:4255, start at 2025-04-04 20:19:04.465509108 +0800 CST m=+2.019269741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:04.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:04.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:04.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431426337979684 TiDB Cluster ID. tikv_gc_leader_uuid 65802f634ec000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:4333, start at 2025-04-04 20:19:04.65746753 +0800 CST m=+2.139216379 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:04.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:04.635 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:04.635 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2685.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5268.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431425791207747 TiDB Cluster ID. tikv_gc_leader_uuid 65802f639780005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:1284, start at 2025-04-04 20:19:04.933207995 +0800 CST m=+2.207954868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:04.947 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:04.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:04.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431425791207747 TiDB Cluster ID. tikv_gc_leader_uuid 65802f639780005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:1284, start at 2025-04-04 20:19:04.933207995 +0800 CST m=+2.207954868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:04.947 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:04.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:04.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431424166645647 TiDB Cluster ID. tikv_gc_leader_uuid 65802f639700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:1364, start at 2025-04-04 20:19:04.93264001 +0800 CST m=+2.122533482 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:04.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:04.924 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:04.924 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.verify_table_eligible not exists for 2-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489431342637464376 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:53.252089887 +0800 CST StartTs:457118615637590019 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035fb8c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118615663804419} {CheckpointTs:457118616044175385 MinTableBarrierTs:457118616044175385 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e83a1f5a-6162-4d94-b916-035528c21426 {"id":"e83a1f5a-6162-4d94-b916-035528c21426","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd261a93 e83a1f5a-6162-4d94-b916-035528c21426 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489431342637464376,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:18:53.252089887+08:00","start-ts":457118615637590019,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118615663804419} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457118616057020442,"min-table-barrier-ts":457118616057020442,"admin-job-type":0} /tidb/cdc/default/default/task/position/e83a1f5a-6162-4d94-b916-035528c21426/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489431342637464376 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:53.252089887 +0800 CST StartTs:457118615637590019 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035fb8c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118615663804419} {CheckpointTs:457118616044175385 MinTableBarrierTs:457118616044175385 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e83a1f5a-6162-4d94-b916-035528c21426 {"id":"e83a1f5a-6162-4d94-b916-035528c21426","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd261a93 e83a1f5a-6162-4d94-b916-035528c21426 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489431342637464376,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:18:53.252089887+08:00","start-ts":457118615637590019,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118615663804419} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457118616057020442,"min-table-barrier-ts":457118616057020442,"admin-job-type":0} /tidb/cdc/default/default/task/position/e83a1f5a-6162-4d94-b916-035528c21426/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489431342637464376 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:18:53.252089887 +0800 CST StartTs:457118615637590019 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035fb8c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118615663804419} {CheckpointTs:457118616044175385 MinTableBarrierTs:457118616044175385 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e83a1f5a-6162-4d94-b916-035528c21426 {"id":"e83a1f5a-6162-4d94-b916-035528c21426","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd261a93 e83a1f5a-6162-4d94-b916-035528c21426 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489431342637464376,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:18:53.252089887+08:00","start-ts":457118615637590019,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118615663804419} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457118616057020442,"min-table-barrier-ts":457118616057020442,"admin-job-type":0} /tidb/cdc/default/default/task/position/e83a1f5a-6162-4d94-b916-035528c21426/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431342637464376 {"id":7489431342637464376,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2678.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431423981950614 TiDB Cluster ID. tikv_gc_leader_uuid 65802f648440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:1418, start at 2025-04-04 20:19:05.877400763 +0800 CST m=+2.391132110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:05.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:05.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:05.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431423981950614 TiDB Cluster ID. tikv_gc_leader_uuid 65802f648440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:1418, start at 2025-04-04 20:19:05.877400763 +0800 CST m=+2.391132110 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:05.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:05.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:05.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431429704097744 TiDB Cluster ID. tikv_gc_leader_uuid 65802f64944000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:1486, start at 2025-04-04 20:19:05.969664634 +0800 CST m=+2.413055920 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:05.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:05.986 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:05.986 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 7.22 secs (557728839 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 4 20:19:07 CST 2025] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.28162818.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='457118619238924289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118619238924289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:19:08 CST 2025] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.27262728.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 20:19:08 CST 2025] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.56005602.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457118619411677188 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118619411677188 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:19:08 CST 2025] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52975299.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2762.out cli tso query --pd=http://127.0.0.1:2379 table test.verify_table_eligible exists + set +x + tso='457118619443134465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118619443134465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:19:09 CST 2025] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.27132715.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: list changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:09 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/deb62fa5-e3b1-4de2-83ad-36505b3885fb {"id":"deb62fa5-e3b1-4de2-83ad-36505b3885fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd659bda deb62fa5-e3b1-4de2-83ad-36505b3885fb /tidb/cdc/default/default/upstream/7489431409970276193 {"id":7489431409970276193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/deb62fa5-e3b1-4de2-83ad-36505b3885fb {"id":"deb62fa5-e3b1-4de2-83ad-36505b3885fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd659bda deb62fa5-e3b1-4de2-83ad-36505b3885fb /tidb/cdc/default/default/upstream/7489431409970276193 {"id":7489431409970276193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/deb62fa5-e3b1-4de2-83ad-36505b3885fb {"id":"deb62fa5-e3b1-4de2-83ad-36505b3885fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd659bda deb62fa5-e3b1-4de2-83ad-36505b3885fb /tidb/cdc/default/default/upstream/7489431409970276193 {"id":7489431409970276193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 table ddl_manager.finish_mark not exists for 2-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2760.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118619907391489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118619907391489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:19:10 CST 2025] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.27992801.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: get changefeed check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:10 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbff29d7-d261-4703-8992-8798ceb0b249 {"id":"cbff29d7-d261-4703-8992-8798ceb0b249","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6c5930 cbff29d7-d261-4703-8992-8798ceb0b249 /tidb/cdc/default/default/upstream/7489431418046017254 {"id":7489431418046017254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbff29d7-d261-4703-8992-8798ceb0b249 {"id":"cbff29d7-d261-4703-8992-8798ceb0b249","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6c5930 cbff29d7-d261-4703-8992-8798ceb0b249 /tidb/cdc/default/default/upstream/7489431418046017254 {"id":7489431418046017254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbff29d7-d261-4703-8992-8798ceb0b249 {"id":"cbff29d7-d261-4703-8992-8798ceb0b249","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6c5930 cbff29d7-d261-4703-8992-8798ceb0b249 /tidb/cdc/default/default/upstream/7489431418046017254 {"id":7489431418046017254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 01fceb8f-a122-466a-a351-c2d705931468 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=01fceb8f-a122-466a-a351-c2d705931468 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 01fceb8f-a122-466a-a351-c2d705931468 -s + info='{ "upstream_id": 7489431418046017254, "namespace": "default", "id": "01fceb8f-a122-466a-a351-c2d705931468", "state": "normal", "checkpoint_tso": 457118620312666114, "checkpoint_time": "2025-04-04 20:19:10.973", "error": null }' + echo '{ "upstream_id": 7489431418046017254, "namespace": "default", "id": "01fceb8f-a122-466a-a351-c2d705931468", "state": "normal", "checkpoint_tso": 457118620312666114, "checkpoint_time": "2025-04-04 20:19:10.973", "error": null }' { "upstream_id": 7489431418046017254, "namespace": "default", "id": "01fceb8f-a122-466a-a351-c2d705931468", "state": "normal", "checkpoint_tso": 457118620312666114, "checkpoint_time": "2025-04-04 20:19:10.973", "error": null } ++ echo '{' '"upstream_id":' 7489431418046017254, '"namespace":' '"default",' '"id":' '"01fceb8f-a122-466a-a351-c2d705931468",' '"state":' '"normal",' '"checkpoint_tso":' 457118620312666114, '"checkpoint_time":' '"2025-04-04' '20:19:10.973",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431418046017254, '"namespace":' '"default",' '"id":' '"01fceb8f-a122-466a-a351-c2d705931468",' '"state":' '"normal",' '"checkpoint_tso":' 457118620312666114, '"checkpoint_time":' '"2025-04-04' '20:19:10.973",' '"error":' null '}' ++ jq -r .error.message start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + message=null + [[ ! null =~ null ]] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15bba63e-aef5-47a7-a678-527ec7ffa8f4 {"id":"15bba63e-aef5-47a7-a678-527ec7ffa8f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd67adf5 15bba63e-aef5-47a7-a678-527ec7ffa8f4 /tidb/cdc/default/default/upstream/7489431417823088815 {"id":7489431417823088815,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15bba63e-aef5-47a7-a678-527ec7ffa8f4 {"id":"15bba63e-aef5-47a7-a678-527ec7ffa8f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd67adf5 15bba63e-aef5-47a7-a678-527ec7ffa8f4 /tidb/cdc/default/default/upstream/7489431417823088815 {"id":7489431417823088815,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15bba63e-aef5-47a7-a678-527ec7ffa8f4 {"id":"15bba63e-aef5-47a7-a678-527ec7ffa8f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769148} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd67adf5 15bba63e-aef5-47a7-a678-527ec7ffa8f4 /tidb/cdc/default/default/upstream/7489431417823088815 {"id":7489431417823088815,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2785.out cli changefeed create --start-ts=457118619238924289 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ + set +x + tso='457118620103213058 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118620103213058 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:19:11 CST 2025] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.28022804.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 3-th check, retry later check diff failed 1-th time, retry later Create changefeed successfully! ID: 337d358e-5ed2-4129-a095-0d1bb7dee526 Info: {"upstream_id":7489431417823088815,"namespace":"default","id":"337d358e-5ed2-4129-a095-0d1bb7dee526","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:11.867270949+08:00","start_ts":457118619238924289,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118619238924289,"checkpoint_ts":457118619238924289,"checkpoint_time":"2025-04-04 20:19:06.877"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0bde5318-727e-4bc2-999a-74fd25850081 {"id":"0bde5318-727e-4bc2-999a-74fd25850081","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6ec2d6 0bde5318-727e-4bc2-999a-74fd25850081 /tidb/cdc/default/default/upstream/7489431425782587193 {"id":7489431425782587193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0bde5318-727e-4bc2-999a-74fd25850081 {"id":"0bde5318-727e-4bc2-999a-74fd25850081","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6ec2d6 0bde5318-727e-4bc2-999a-74fd25850081 /tidb/cdc/default/default/upstream/7489431425782587193 {"id":7489431425782587193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0bde5318-727e-4bc2-999a-74fd25850081 {"id":"0bde5318-727e-4bc2-999a-74fd25850081","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6ec2d6 0bde5318-727e-4bc2-999a-74fd25850081 /tidb/cdc/default/default/upstream/7489431425782587193 {"id":7489431425782587193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5656.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:12 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f74e927-0baa-47b4-9d85-3eb8786c178a {"id":"6f74e927-0baa-47b4-9d85-3eb8786c178a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6689ea 6f74e927-0baa-47b4-9d85-3eb8786c178a /tidb/cdc/default/default/upstream/7489431413369416632 {"id":7489431413369416632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f74e927-0baa-47b4-9d85-3eb8786c178a {"id":"6f74e927-0baa-47b4-9d85-3eb8786c178a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6689ea 6f74e927-0baa-47b4-9d85-3eb8786c178a /tidb/cdc/default/default/upstream/7489431413369416632 {"id":7489431413369416632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f74e927-0baa-47b4-9d85-3eb8786c178a {"id":"6f74e927-0baa-47b4-9d85-3eb8786c178a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd6689ea 6f74e927-0baa-47b4-9d85-3eb8786c178a /tidb/cdc/default/default/upstream/7489431413369416632 {"id":7489431413369416632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5351.out cli changefeed create --start-ts=457118619411677188 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: a0425e96-3fb4-49be-b53d-1187490e8d92 Info: {"upstream_id":7489431425782587193,"namespace":"default","id":"a0425e96-3fb4-49be-b53d-1187490e8d92","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:12.259949713+08:00","start_ts":457118620631433218,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118620631433218,"checkpoint_ts":457118620631433218,"checkpoint_time":"2025-04-04 20:19:12.189"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: 953c2017-31e2-44d4-bc74-edbc682be1a6 Info: {"upstream_id":7489431413369416632,"namespace":"default","id":"953c2017-31e2-44d4-bc74-edbc682be1a6","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:12.454190326+08:00","start_ts":457118619411677188,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118619411677188,"checkpoint_ts":457118619411677188,"checkpoint_time":"2025-04-04 20:19:07.536"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:12 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/990dfa62-cc8d-4722-bc27-e08a022d9770 {"id":"990dfa62-cc8d-4722-bc27-e08a022d9770","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd68f5e3 990dfa62-cc8d-4722-bc27-e08a022d9770 /tidb/cdc/default/default/upstream/7489431419166727409 {"id":7489431419166727409,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/990dfa62-cc8d-4722-bc27-e08a022d9770 {"id":"990dfa62-cc8d-4722-bc27-e08a022d9770","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd68f5e3 990dfa62-cc8d-4722-bc27-e08a022d9770 /tidb/cdc/default/default/upstream/7489431419166727409 {"id":7489431419166727409,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/990dfa62-cc8d-4722-bc27-e08a022d9770 {"id":"990dfa62-cc8d-4722-bc27-e08a022d9770","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd68f5e3 990dfa62-cc8d-4722-bc27-e08a022d9770 /tidb/cdc/default/default/upstream/7489431419166727409 {"id":7489431419166727409,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 55bdeec7-0be1-4a91-8763-1315726a7e8f Info: {"upstream_id":7489431419166727409,"namespace":"default","id":"55bdeec7-0be1-4a91-8763-1315726a7e8f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:12.316110238+08:00","start_ts":457118619443134465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118619443134465,"checkpoint_ts":457118619443134465,"checkpoint_time":"2025-04-04 20:19:07.656"} check diff failed 2-th time, retry later + set +x succeed to verify meta placement rules pass test: pause changefeed TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later table ddl_manager.finish_mark not exists for 4-th check, retry later + set +x go: downloading go1.23.4 (linux/amd64) + set +x table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:19:11 CST 2025] <<<<<< run test case sink_retry success! >>>>>> table drop_tables.c not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:13 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d915c75d-5124-4b17-bdae-e551ce095428 {"id":"d915c75d-5124-4b17-bdae-e551ce095428","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd77b0e2 d915c75d-5124-4b17-bdae-e551ce095428 /tidb/cdc/default/default/upstream/7489431425791207747 {"id":7489431425791207747,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d915c75d-5124-4b17-bdae-e551ce095428 {"id":"d915c75d-5124-4b17-bdae-e551ce095428","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd77b0e2 d915c75d-5124-4b17-bdae-e551ce095428 /tidb/cdc/default/default/upstream/7489431425791207747 {"id":7489431425791207747,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d915c75d-5124-4b17-bdae-e551ce095428 {"id":"d915c75d-5124-4b17-bdae-e551ce095428","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd77b0e2 d915c75d-5124-4b17-bdae-e551ce095428 /tidb/cdc/default/default/upstream/7489431425791207747 {"id":7489431425791207747,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2853.out cli changefeed create --start-ts=457118619907391489 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff failed 2-th time, retry later Create changefeed successfully! ID: 008d7030-4045-4608-9c91-a97c841b429f Info: {"upstream_id":7489431425791207747,"namespace":"default","id":"008d7030-4045-4608-9c91-a97c841b429f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:14.391513543+08:00","start_ts":457118619907391489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118619907391489,"checkpoint_ts":457118619907391489,"checkpoint_time":"2025-04-04 20:19:09.427"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... pass test: update changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:14 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8471a7d1-9565-4e90-81f2-48e664cd05ca {"id":"8471a7d1-9565-4e90-81f2-48e664cd05ca","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd7682ea 8471a7d1-9565-4e90-81f2-48e664cd05ca /tidb/cdc/default/default/upstream/7489431423981950614 {"id":7489431423981950614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8471a7d1-9565-4e90-81f2-48e664cd05ca {"id":"8471a7d1-9565-4e90-81f2-48e664cd05ca","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd7682ea 8471a7d1-9565-4e90-81f2-48e664cd05ca /tidb/cdc/default/default/upstream/7489431423981950614 {"id":7489431423981950614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8471a7d1-9565-4e90-81f2-48e664cd05ca {"id":"8471a7d1-9565-4e90-81f2-48e664cd05ca","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769151} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bd7682ea 8471a7d1-9565-4e90-81f2-48e664cd05ca /tidb/cdc/default/default/upstream/7489431423981950614 {"id":7489431423981950614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2859.out cli changefeed create --start-ts=457118620103213058 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic Create changefeed successfully! ID: dfa0dbeb-1ec4-44ac-9b2e-49a9a49f3a9c Info: {"upstream_id":7489431423981950614,"namespace":"default","id":"dfa0dbeb-1ec4-44ac-9b2e-49a9a49f3a9c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2025-04-04T20:19:15.143750131+08:00","start_ts":457118620103213058,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118620103213058,"checkpoint_ts":457118620103213058,"checkpoint_time":"2025-04-04 20:19:10.174"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 5-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7489431409970276193 457118621144449032 457118619964801031 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x go: downloading go1.23.4 (linux/amd64) table cdc_tiflash_test.multi_data_type exists TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later check diff failed 1-th time, retry later check diff successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 6.83 secs (589867138 bytes/sec) [Pipeline] { [Pipeline] cache run task successfully check_changefeed_state http://127.0.0.1:2379 3288057e-238f-4212-8be1-b733403f68ea stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=3288057e-238f-4212-8be1-b733403f68ea + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 3288057e-238f-4212-8be1-b733403f68ea -s + set +x go: downloading go1.23.4 (linux/amd64) table drop_tables.c not exists for 2-th check, retry later + info='{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "stopped", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null }' + echo '{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "stopped", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null }' { "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "stopped", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null } ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"3288057e-238f-4212-8be1-b733403f68ea",' '"state":' '"stopped",' '"checkpoint_tso":' 457118621406593045, '"checkpoint_time":' '"2025-04-04' '20:19:15.146",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"3288057e-238f-4212-8be1-b733403f68ea",' '"state":' '"stopped",' '"checkpoint_tso":' 457118621406593045, '"checkpoint_time":' '"2025-04-04' '20:19:15.146",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7489431409970276193 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... pass test: resume changefeed table ddl_manager.finish_mark not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:19:17 CST 2025] <<<<<< run test case processor_err_chan success! >>>>>> check data successfully wait process cdc.test exit for 1-th time... table drop_tables.c not exists for 3-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... pass test: rebalance table go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:19:19 CST 2025] <<<<<< run test case simple success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:19:19 CST 2025] <<<<<< run test case tiflash success! >>>>>> table ddl_manager.finish_mark not exists for 7-th check, retry later run task successfully pass test: list processors ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 3288057e-238f-4212-8be1-b733403f68ea normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=3288057e-238f-4212-8be1-b733403f68ea + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 3288057e-238f-4212-8be1-b733403f68ea -s + info='{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "normal", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null }' + echo '{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "normal", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null }' { "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "normal", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null } ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"3288057e-238f-4212-8be1-b733403f68ea",' '"state":' '"normal",' '"checkpoint_tso":' 457118621406593045, '"checkpoint_time":' '"2025-04-04' '20:19:15.146",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"3288057e-238f-4212-8be1-b733403f68ea",' '"state":' '"normal",' '"checkpoint_tso":' 457118621406593045, '"checkpoint_time":' '"2025-04-04' '20:19:15.146",' '"error":' null '}' ++ jq -r .error.message table drop_tables.c not exists for 4-th check, retry later + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7489431409970276193 457118621406593044 457118621406593045 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 run task successfully go: downloading golang.org/x/text v0.21.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading golang.org/x/net v0.33.0 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/otiai10/copy v1.2.0 check_changefeed_state http://127.0.0.1:2379 3288057e-238f-4212-8be1-b733403f68ea stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=3288057e-238f-4212-8be1-b733403f68ea + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 3288057e-238f-4212-8be1-b733403f68ea -s + info='{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "stopped", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null }' + echo '{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "stopped", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null }' { "upstream_id": 7489431409970276193, "namespace": "default", "id": "3288057e-238f-4212-8be1-b733403f68ea", "state": "stopped", "checkpoint_tso": 457118621406593045, "checkpoint_time": "2025-04-04 20:19:15.146", "error": null } ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"3288057e-238f-4212-8be1-b733403f68ea",' '"state":' '"stopped",' '"checkpoint_tso":' 457118621406593045, '"checkpoint_time":' '"2025-04-04' '20:19:15.146",' '"error":' null '}' ++ jq -r .state go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 table ddl_manager.finish_mark not exists for 8-th check, retry later go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading google.golang.org/grpc v1.64.0 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"3288057e-238f-4212-8be1-b733403f68ea",' '"state":' '"stopped",' '"checkpoint_tso":' 457118621406593045, '"checkpoint_time":' '"2025-04-04' '20:19:15.146",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 2d502511-5d3e-4457-96ef-9a04c062ef4d normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=2d502511-5d3e-4457-96ef-9a04c062ef4d + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2d502511-5d3e-4457-96ef-9a04c062ef4d -s go: downloading golang.org/x/time v0.7.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431486582644769 TiDB Cluster ID. tikv_gc_leader_uuid 65802f72390000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:4024, start at 2025-04-04 20:19:19.931565677 +0800 CST m=+2.309128844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:19.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:19.908 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:19.908 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431486582644769 TiDB Cluster ID. tikv_gc_leader_uuid 65802f72390000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:4024, start at 2025-04-04 20:19:19.931565677 +0800 CST m=+2.309128844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:19.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:19.908 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:19.908 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431491974329448 TiDB Cluster ID. tikv_gc_leader_uuid 65802f7251c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:4097, start at 2025-04-04 20:19:20.031746567 +0800 CST m=+2.325681178 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:20.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:20.007 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:20.007 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-semver v0.3.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading golang.org/x/sys v0.28.0 + info='{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "2d502511-5d3e-4457-96ef-9a04c062ef4d", "state": "normal", "checkpoint_tso": 457118623162957833, "checkpoint_time": "2025-04-04 20:19:21.846", "error": null }' + echo '{ "upstream_id": 7489431409970276193, "namespace": "default", "id": "2d502511-5d3e-4457-96ef-9a04c062ef4d", "state": "normal", "checkpoint_tso": 457118623162957833, "checkpoint_time": "2025-04-04 20:19:21.846", "error": null }' { "upstream_id": 7489431409970276193, "namespace": "default", "id": "2d502511-5d3e-4457-96ef-9a04c062ef4d", "state": "normal", "checkpoint_tso": 457118623162957833, "checkpoint_time": "2025-04-04 20:19:21.846", "error": null } ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"2d502511-5d3e-4457-96ef-9a04c062ef4d",' '"state":' '"normal",' '"checkpoint_tso":' 457118623162957833, '"checkpoint_time":' '"2025-04-04' '20:19:21.846",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431409970276193, '"namespace":' '"default",' '"id":' '"2d502511-5d3e-4457-96ef-9a04c062ef4d",' '"state":' '"normal",' '"checkpoint_tso":' 457118623162957833, '"checkpoint_time":' '"2025-04-04' '20:19:21.846",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7489431409970276193 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 table drop_tables.c not exists for 5-th check, retry later go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading golang.org/x/net v0.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.21.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading golang.org/x/text v0.21.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.10.0 go: downloading google.golang.org/grpc v1.64.0 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/google/btree v1.1.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/robfig/cron v1.2.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible pass test: get processor go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/go-logr/stdr v1.2.2 table ddl_manager.finish_mark not exists for 9-th check, retry later go: downloading golang.org/x/time v0.7.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.61.0 table drop_tables.c exists go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 check diff successfully go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5346.out cli tso query --pd=http://127.0.0.1:2379 pass test: move table go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 wait process cdc.test exit for 1-th time... go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 wait process cdc.test exit for 2-th time... run task successfully start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da table ddl_manager.finish_mark not exists for 10-th check, retry later Changefeed remove successfully. ID: 3288057e-238f-4212-8be1-b733403f68ea CheckpointTs: 457118621406593045 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/go-logr/logr v1.4.1 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:19:25 CST 2025] <<<<<< run test case drop_many_tables success! >>>>>> check_safepoint_forward http://127.0.0.1:2379 7489431409970276193 457118621406593044 457118623162957833 457118621406593045 + set +x + tso='457118623821463553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118623821463553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x pass test: set log level go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.05 secs (500652234 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading k8s.io/apimachinery v0.29.11 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 run task successfully [Fri Apr 4 20:19:26 CST 2025] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.53955397.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b Changefeed remove successfully. ID: 2d502511-5d3e-4457-96ef-9a04c062ef4d CheckpointTs: 457118624303022085 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7489431409970276193 run task successfully pass test: remove changefeed table ddl_manager.finish_mark not exists for 11-th check, retry later go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... go: downloading github.com/pierrec/lz4 v2.6.1+incompatible wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:19:28 CST 2025] <<<<<< run test case gc_safepoint success! >>>>>> pass test: resign owner go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 12-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: get tso go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:29 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e55c1bcb-2693-4a54-8cef-2a864c411378 {"id":"e55c1bcb-2693-4a54-8cef-2a864c411378","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769167} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bda7dcfe e55c1bcb-2693-4a54-8cef-2a864c411378 /tidb/cdc/default/default/upstream/7489431486582644769 {"id":7489431486582644769,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e55c1bcb-2693-4a54-8cef-2a864c411378 {"id":"e55c1bcb-2693-4a54-8cef-2a864c411378","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769167} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bda7dcfe e55c1bcb-2693-4a54-8cef-2a864c411378 /tidb/cdc/default/default/upstream/7489431486582644769 {"id":7489431486582644769,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e55c1bcb-2693-4a54-8cef-2a864c411378 {"id":"e55c1bcb-2693-4a54-8cef-2a864c411378","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769167} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bda7dcfe e55c1bcb-2693-4a54-8cef-2a864c411378 /tidb/cdc/default/default/upstream/7489431486582644769 {"id":7489431486582644769,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5455.out cli changefeed create --start-ts=457118623821463553 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ go: downloading github.com/jmespath/go-jmespath v0.4.0 Create changefeed successfully! ID: b8305ce3-619f-4ac8-9097-a2380cf2a66a Info: {"upstream_id":7489431486582644769,"namespace":"default","id":"b8305ce3-619f-4ac8-9097-a2380cf2a66a","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:30.378310262+08:00","start_ts":457118623821463553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118623821463553,"checkpoint_ts":457118623821463553,"checkpoint_time":"2025-04-04 20:19:24.358"} PASS Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... pass test: verify table start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 13-th check, retry later + set +x start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... pass test: create changefeed v2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 14-th check, retry later pass test: remove changefeed v2 Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431545386418571 TiDB Cluster ID. tikv_gc_leader_uuid 65802f7ee740008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:4017, start at 2025-04-04 20:19:32.900685425 +0800 CST m=+2.306566640 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:32.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:32.893 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:32.893 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431545386418571 TiDB Cluster ID. tikv_gc_leader_uuid 65802f7ee740008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:4017, start at 2025-04-04 20:19:32.900685425 +0800 CST m=+2.306566640 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:32.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:32.893 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:32.893 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431547571866584 TiDB Cluster ID. tikv_gc_leader_uuid 65802f7f000000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:4085, start at 2025-04-04 20:19:33.033144624 +0800 CST m=+2.379635488 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:33.040 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:33.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:33.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock table ddl_manager.finish_mark not exists for 15-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.23 secs (489325345 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5354.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 16-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:19:38 CST 2025] <<<<<< run test case http_api_tls success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='457118627225403393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118627225403393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.port"="4000" "mysql.user"="root" "requestdistribution"="uniform" "operationcount"="0" "insertproportion"="0" "dotransactions"="false" "recordcount"="20" "readproportion"="0" "workload"="core" "threadcount"="4" "mysql.host"="127.0.0.1" "scanproportion"="0" "updateproportion"="0" "mysql.db"="changefeed_error" "readallfields"="true" ********************************************** Run finished, takes 10.993724ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2607.0, Avg(us): 1874, Min(us): 1027, Max(us): 4793, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 20:19:38 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.54265428.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 17-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431570020861861 TiDB Cluster ID. tikv_gc_leader_uuid 65802f850540009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:4165, start at 2025-04-04 20:19:39.161399462 +0800 CST m=+2.562746930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:39.169 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:39.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:39.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431570020861861 TiDB Cluster ID. tikv_gc_leader_uuid 65802f850540009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:4165, start at 2025-04-04 20:19:39.161399462 +0800 CST m=+2.562746930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:39.169 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:39.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:39.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431570219599680 TiDB Cluster ID. tikv_gc_leader_uuid 65802f84ec4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:4242, start at 2025-04-04 20:19:39.083954601 +0800 CST m=+2.413481803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:39.093 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:39.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:39.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431575505711220 TiDB Cluster ID. tikv_gc_leader_uuid 65802f85c08000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:3869, start at 2025-04-04 20:19:39.951418091 +0800 CST m=+2.624282956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:39.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:39.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:39.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431575505711220 TiDB Cluster ID. tikv_gc_leader_uuid 65802f85c08000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:3869, start at 2025-04-04 20:19:39.951418091 +0800 CST m=+2.624282956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:39.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:39.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:39.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431568073772693 TiDB Cluster ID. tikv_gc_leader_uuid 65802f872c80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:3950, start at 2025-04-04 20:19:41.362610919 +0800 CST m=+3.982474145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:41.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:41.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:41.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 18-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:41 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 {"id":"923af241-25c8-4fa0-a0fd-9b87e06c2613","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769179} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeacf5 923af241-25c8-4fa0-a0fd-9b87e06c2613 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 {"id":"923af241-25c8-4fa0-a0fd-9b87e06c2613","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769179} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeacf5 923af241-25c8-4fa0-a0fd-9b87e06c2613 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 {"id":"923af241-25c8-4fa0-a0fd-9b87e06c2613","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769179} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeacf5 923af241-25c8-4fa0-a0fd-9b87e06c2613 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5480.out cli changefeed create --start-ts=457118627225403393 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7489431545386418571,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:19:42.400295275+08:00","start_ts":457118627225403393,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118627225403393,"checkpoint_ts":457118627225403393,"checkpoint_time":"2025-04-04 20:19:37.343"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Fri Apr 4 20:19:43 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431573972322351 TiDB Cluster ID. tikv_gc_leader_uuid 65802f86ae00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:3844, start at 2025-04-04 20:19:40.890522054 +0800 CST m=+2.249376665 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:40.944 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:40.936 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:40.936 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431573972322351 TiDB Cluster ID. tikv_gc_leader_uuid 65802f86ae00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:3844, start at 2025-04-04 20:19:40.890522054 +0800 CST m=+2.249376665 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:40.944 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:40.936 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:40.936 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431581944103187 TiDB Cluster ID. tikv_gc_leader_uuid 65802f86dec000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:3914, start at 2025-04-04 20:19:41.068772419 +0800 CST m=+2.369003795 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:41.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:41.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:41.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.54915493.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 4 20:19:43 CST 2025] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.53315333.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 19-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457118627225403393, "checkpoint_time": "2025-04-04 20:19:37.343", "error": { "time": "2025-04-04T20:19:42.511078216+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at 457118627225403393" } }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457118627225403393, "checkpoint_time": "2025-04-04 20:19:37.343", "error": { "time": "2025-04-04T20:19:42.511078216+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at 457118627225403393" } }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457118627225403393, "checkpoint_time": "2025-04-04 20:19:37.343", "error": { "time": "2025-04-04T20:19:42.511078216+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at 457118627225403393" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 457118627225403393, '"checkpoint_time":' '"2025-04-04' '20:19:37.343",' '"error":' '{' '"time":' '"2025-04-04T20:19:42.511078216+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at '457118627225403393"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 457118627225403393, '"checkpoint_time":' '"2025-04-04' '20:19:37.343",' '"error":' '{' '"time":' '"2025-04-04T20:19:42.511078216+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at '457118627225403393"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at 457118627225403393' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457118627225403093 is earlier than or equal to GC safepoint at 457118627225403393 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5534.out cli changefeed resume -c changefeed-error + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.5312.out cli tso query --pd=http://127.0.0.1:2379 Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 6.68 secs (603242813 bytes/sec) [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] cache table ddl_manager.finish_mark not exists for 20-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:46 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fea249c-0f1d-4605-ad4f-b056299c8bd4 {"id":"5fea249c-0f1d-4605-ad4f-b056299c8bd4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769183} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf42ed6 5fea249c-0f1d-4605-ad4f-b056299c8bd4 /tidb/cdc/default/default/upstream/7489431570020861861 {"id":7489431570020861861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fea249c-0f1d-4605-ad4f-b056299c8bd4 {"id":"5fea249c-0f1d-4605-ad4f-b056299c8bd4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769183} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf42ed6 5fea249c-0f1d-4605-ad4f-b056299c8bd4 /tidb/cdc/default/default/upstream/7489431570020861861 {"id":7489431570020861861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fea249c-0f1d-4605-ad4f-b056299c8bd4 {"id":"5fea249c-0f1d-4605-ad4f-b056299c8bd4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769183} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf42ed6 5fea249c-0f1d-4605-ad4f-b056299c8bd4 /tidb/cdc/default/default/upstream/7489431570020861861 {"id":7489431570020861861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "readproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="4" "updateproportion"="0" "scanproportion"="0" "workload"="core" "mysql.port"="4000" "mysql.db"="changefeed_reconstruct" "readallfields"="true" "mysql.host"="127.0.0.1" "recordcount"="50" "mysql.user"="root" ********************************************** Run finished, takes 22.697418ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2906.8, Avg(us): 1804, Min(us): 1122, Max(us): 6164, 95th(us): 7000, 99th(us): 7000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457118629325963265 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118629325963265 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:46 GMT < Content-Length: 864 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6e8b399-67be-4b91-8647-a1996ea10b9e {"id":"b6e8b399-67be-4b91-8647-a1996ea10b9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769184} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf8dbe7 b6e8b399-67be-4b91-8647-a1996ea10b9e /tidb/cdc/default/default/upstream/7489431575505711220 {"id":7489431575505711220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6e8b399-67be-4b91-8647-a1996ea10b9e {"id":"b6e8b399-67be-4b91-8647-a1996ea10b9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769184} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf8dbe7 b6e8b399-67be-4b91-8647-a1996ea10b9e /tidb/cdc/default/default/upstream/7489431575505711220 {"id":7489431575505711220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6e8b399-67be-4b91-8647-a1996ea10b9e {"id":"b6e8b399-67be-4b91-8647-a1996ea10b9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769184} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf8dbe7 b6e8b399-67be-4b91-8647-a1996ea10b9e /tidb/cdc/default/default/upstream/7489431575505711220 {"id":7489431575505711220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 0cd0a779-4713-4599-8b05-eb312fca1cad Info: {"upstream_id":7489431575505711220,"namespace":"default","id":"0cd0a779-4713-4599-8b05-eb312fca1cad","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:19:46.865443289+08:00","start_ts":457118629693227014,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118629693227014,"checkpoint_ts":457118629693227014,"checkpoint_time":"2025-04-04 20:19:46.757"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431597297651469 TiDB Cluster ID. tikv_gc_leader_uuid 65802f8af600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:6449, start at 2025-04-04 20:19:45.259129919 +0800 CST m=+2.322978211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:45.267 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:45.240 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:45.240 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431597297651469 TiDB Cluster ID. tikv_gc_leader_uuid 65802f8af600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:6449, start at 2025-04-04 20:19:45.259129919 +0800 CST m=+2.322978211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:45.267 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:45.240 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:45.240 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431596106441202 TiDB Cluster ID. tikv_gc_leader_uuid 65802f8b0e0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:6525, start at 2025-04-04 20:19:45.36145232 +0800 CST m=+2.328945098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:45.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:45.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:45.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x table changefeed_error.usertable not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } try a VALID cdc server command [Fri Apr 4 20:19:47 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.53545356.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 21-th check, retry later split_and_random_merge scale: 20 *************************** 1. row *************************** count(distinct region_id): 1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later [Fri Apr 4 20:19:49 CST 2025] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.77937795.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:50 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7a5cc03-b342-4917-a831-198969b9796f {"id":"c7a5cc03-b342-4917-a831-198969b9796f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769187} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdfa14f3 c7a5cc03-b342-4917-a831-198969b9796f /tidb/cdc/default/default/upstream/7489431573972322351 {"id":7489431573972322351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7a5cc03-b342-4917-a831-198969b9796f {"id":"c7a5cc03-b342-4917-a831-198969b9796f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769187} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdfa14f3 c7a5cc03-b342-4917-a831-198969b9796f /tidb/cdc/default/default/upstream/7489431573972322351 {"id":7489431573972322351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7a5cc03-b342-4917-a831-198969b9796f {"id":"c7a5cc03-b342-4917-a831-198969b9796f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769187} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdfa14f3 c7a5cc03-b342-4917-a831-198969b9796f /tidb/cdc/default/default/upstream/7489431573972322351 {"id":7489431573972322351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431612586145425 TiDB Cluster ID. tikv_gc_leader_uuid 65802f8eaf80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:4439, start at 2025-04-04 20:19:49.060406797 +0800 CST m=+2.368194776 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:49.068 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:49.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:49.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431612586145425 TiDB Cluster ID. tikv_gc_leader_uuid 65802f8eaf80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:4439, start at 2025-04-04 20:19:49.060406797 +0800 CST m=+2.368194776 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:49.068 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:49.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:49.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431616456480432 TiDB Cluster ID. tikv_gc_leader_uuid 65802f8eaec000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:4508, start at 2025-04-04 20:19:49.08860986 +0800 CST m=+2.329253973 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:21:49.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:19:49.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:09:49.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 5496 exit for 1-th time... check diff successfully ***************** properties ***************** "readproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "mysql.port"="4000" "operationcount"="0" "dotransactions"="false" "mysql.db"="changefeed_error" "updateproportion"="0" "mysql.user"="root" "threadcount"="4" "requestdistribution"="uniform" "recordcount"="20" "readallfields"="true" "insertproportion"="0" ********************************************** Run finished, takes 4.074685ms INSERT - Takes(s): 0.0, Count: 20, OPS: 7199.2, Avg(us): 689, Min(us): 406, Max(us): 1717, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"923af241-25c8-4fa0-a0fd-9b87e06c2613","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769179} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 + [[ /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later wait process 5496 exit for 2-th time... table ddl_manager.finish_mark not exists for 23-th check, retry later wait process 5496 exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:52 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c764949a-78f5-4a02-9090-a93c6774ce0d {"id":"c764949a-78f5-4a02-9090-a93c6774ce0d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769189} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be12d2d5 c764949a-78f5-4a02-9090-a93c6774ce0d /tidb/cdc/default/default/upstream/7489431597297651469 {"id":7489431597297651469,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c764949a-78f5-4a02-9090-a93c6774ce0d {"id":"c764949a-78f5-4a02-9090-a93c6774ce0d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769189} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be12d2d5 c764949a-78f5-4a02-9090-a93c6774ce0d /tidb/cdc/default/default/upstream/7489431597297651469 {"id":7489431597297651469,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c764949a-78f5-4a02-9090-a93c6774ce0d {"id":"c764949a-78f5-4a02-9090-a93c6774ce0d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769189} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be12d2d5 c764949a-78f5-4a02-9090-a93c6774ce0d /tidb/cdc/default/default/upstream/7489431597297651469 {"id":7489431597297651469,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5496) - No such process wait process 5496 exit for 4-th time... process 5496 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 20:19:52 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.57515753.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_stop_delay.t not exists for 1-th check, retry later [Fri Apr 4 20:19:53 CST 2025] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.57575759.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 24-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 + [[ /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later Verifying downstream PD is started... table processor_stop_delay.t exists check diff failed 1-th time, retry later *************************** 1. row *************************** count(distinct region_id): 20 table ddl_manager.finish_mark not exists for 25-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:55 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {UpstreamID:7489431570020861861 Namespace:default ID:cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:19:46.259536476 +0800 CST StartTs:457118629533319172 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003aa4cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118629559533571} {CheckpointTs:457118630463930372 MinTableBarrierTs:457118632036794372 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118630463930372, checkpointTs: 457118630463930372, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/093f0574-f53e-4b5f-89c9-363a2bce6351 {"id":"093f0574-f53e-4b5f-89c9-363a2bce6351","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769193} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf42f77 093f0574-f53e-4b5f-89c9-363a2bce6351 /tidb/cdc/default/default/changefeed/info/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"upstream-id":7489431570020861861,"namespace":"default","changefeed-id":"cfc46c0e-273b-4788-b9c7-e9f5ed357eb3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:19:46.259536476+08:00","start-ts":457118629533319172,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118629559533571} /tidb/cdc/default/default/changefeed/status/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"checkpoint-ts":457118630463930372,"min-table-barrier-ts":457118632036794372,"admin-job-type":0} /tidb/cdc/default/default/task/position/093f0574-f53e-4b5f-89c9-363a2bce6351/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431570020861861 {"id":7489431570020861861,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {UpstreamID:7489431570020861861 Namespace:default ID:cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:19:46.259536476 +0800 CST StartTs:457118629533319172 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003aa4cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118629559533571} {CheckpointTs:457118630463930372 MinTableBarrierTs:457118632036794372 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118630463930372, checkpointTs: 457118630463930372, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/093f0574-f53e-4b5f-89c9-363a2bce6351 {"id":"093f0574-f53e-4b5f-89c9-363a2bce6351","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769193} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf42f77 093f0574-f53e-4b5f-89c9-363a2bce6351 /tidb/cdc/default/default/changefeed/info/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"upstream-id":7489431570020861861,"namespace":"default","changefeed-id":"cfc46c0e-273b-4788-b9c7-e9f5ed357eb3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:19:46.259536476+08:00","start-ts":457118629533319172,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118629559533571} /tidb/cdc/default/default/changefeed/status/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"checkpoint-ts":457118630463930372,"min-table-barrier-ts":457118632036794372,"admin-job-type":0} /tidb/cdc/default/default/task/position/093f0574-f53e-4b5f-89c9-363a2bce6351/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431570020861861 {"id":7489431570020861861,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {UpstreamID:7489431570020861861 Namespace:default ID:cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:19:46.259536476 +0800 CST StartTs:457118629533319172 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003aa4cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118629559533571} {CheckpointTs:457118630463930372 MinTableBarrierTs:457118632036794372 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118630463930372, checkpointTs: 457118630463930372, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/093f0574-f53e-4b5f-89c9-363a2bce6351 {"id":"093f0574-f53e-4b5f-89c9-363a2bce6351","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769193} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bdf42f77 093f0574-f53e-4b5f-89c9-363a2bce6351 /tidb/cdc/default/default/changefeed/info/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"upstream-id":7489431570020861861,"namespace":"default","changefeed-id":"cfc46c0e-273b-4788-b9c7-e9f5ed357eb3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:19:46.259536476+08:00","start-ts":457118629533319172,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118629559533571} /tidb/cdc/default/default/changefeed/status/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"checkpoint-ts":457118630463930372,"min-table-barrier-ts":457118632036794372,"admin-job-type":0} /tidb/cdc/default/default/task/position/093f0574-f53e-4b5f-89c9-363a2bce6351/cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431570020861861 {"id":7489431570020861861,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:19:56 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4064a39f-fbf1-4c81-bce9-f62949fe28a3 {"id":"4064a39f-fbf1-4c81-bce9-f62949fe28a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769193} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be1d6cd9 4064a39f-fbf1-4c81-bce9-f62949fe28a3 /tidb/cdc/default/default/upstream/7489431612586145425 {"id":7489431612586145425,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4064a39f-fbf1-4c81-bce9-f62949fe28a3 {"id":"4064a39f-fbf1-4c81-bce9-f62949fe28a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769193} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be1d6cd9 4064a39f-fbf1-4c81-bce9-f62949fe28a3 /tidb/cdc/default/default/upstream/7489431612586145425 {"id":7489431612586145425,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4064a39f-fbf1-4c81-bce9-f62949fe28a3 {"id":"4064a39f-fbf1-4c81-bce9-f62949fe28a3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769193} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be1d6cd9 4064a39f-fbf1-4c81-bce9-f62949fe28a3 /tidb/cdc/default/default/upstream/7489431612586145425 {"id":7489431612586145425,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.92 secs (451598071 bytes/sec) [Pipeline] { [Pipeline] cache check diff successfully "id": "093f0574-f53e-4b5f-89c9-363a2bce6351", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 26-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 + [[ /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/923af241-25c8-4fa0-a0fd-9b87e06c2613 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later check diff failed 1-th time, retry later table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 093f0574-f53e-4b5f-89c9-363a2bce6351 check_processor_table_count http://127.0.0.1:2379 cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 093f0574-f53e-4b5f-89c9-363a2bce6351 1 run task successfully check_processor_table_count http://127.0.0.1:2379 cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 093f0574-f53e-4b5f-89c9-363a2bce6351 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 27-th check, retry later split_and_random_merge scale: 40 table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later *************************** 1. row *************************** count(distinct region_id): 1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check_processor_table_count http://127.0.0.1:2379 cfc46c0e-273b-4788-b9c7-e9f5ed357eb3 093f0574-f53e-4b5f-89c9-363a2bce6351 0 table ddl_manager.finish_mark not exists for 28-th check, retry later run task successfully ***************** properties ***************** "insertproportion"="0" "recordcount"="50" "dotransactions"="false" "readallfields"="true" "mysql.port"="4000" "workload"="core" "updateproportion"="0" "readproportion"="0" "scanproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "threadcount"="4" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 26.430278ms INSERT - Takes(s): 0.0, Count: 47, OPS: 3096.2, Avg(us): 1922, Min(us): 1049, Max(us): 11036, 95th(us): 11000, 99th(us): 12000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table changefeed_pause_resume.t3 exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431659457890943 TiDB Cluster ID. tikv_gc_leader_uuid 65802f9adf4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:7406, start at 2025-04-04 20:20:01.573771119 +0800 CST m=+2.944847573 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:01.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:01.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:01.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431659457890943 TiDB Cluster ID. tikv_gc_leader_uuid 65802f9adf4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:7406, start at 2025-04-04 20:20:01.573771119 +0800 CST m=+2.944847573 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:01.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:01.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:01.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431664689864890 TiDB Cluster ID. tikv_gc_leader_uuid 65802f9af6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:7480, start at 2025-04-04 20:20:01.652250619 +0800 CST m=+2.959580716 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:01.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:01.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:01.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 3-th time, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 29-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 4 20:20:03 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57605762.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later [Fri Apr 4 20:20:05 CST 2025] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.88148816.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully table ddl_manager.finish_mark not exists for 30-th check, retry later wait process cdc.test exit for 1-th time... check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.18 secs (492546258 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/637f9291-0742-4393-994e-d9bd37ba4530 {"id":"637f9291-0742-4393-994e-d9bd37ba4530","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769204} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeae06 637f9291-0742-4393-994e-d9bd37ba4530 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489431545386418571,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:19:42.400295275+08:00","start-ts":457118627225403393,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-04T20:20:04.352547279+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118634303291394} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457118630711918595,"min-table-barrier-ts":457118630711918595,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/637f9291-0742-4393-994e-d9bd37ba4530 {"id":"637f9291-0742-4393-994e-d9bd37ba4530","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769204} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeae06 637f9291-0742-4393-994e-d9bd37ba4530 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489431545386418571,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:19:42.400295275+08:00","start-ts":457118627225403393,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-04T20:20:04.352547279+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118634303291394} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457118630711918595,"min-table-barrier-ts":457118630711918595,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/637f9291-0742-4393-994e-d9bd37ba4530 {"id":"637f9291-0742-4393-994e-d9bd37ba4530","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769204} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeae06 637f9291-0742-4393-994e-d9bd37ba4530 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489431545386418571,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:19:42.400295275+08:00","start-ts":457118627225403393,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-04T20:20:04.352547279+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118634303291394} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457118630711918595,"min-table-barrier-ts":457118630711918595,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457118630711918595, "checkpoint_time": "2025-04-04 20:19:50.643", "error": { "time": "2025-04-04T20:20:04.352547279+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457118630711918595, "checkpoint_time": "2025-04-04 20:19:50.643", "error": { "time": "2025-04-04T20:20:04.352547279+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457118630711918595, "checkpoint_time": "2025-04-04 20:19:50.643", "error": { "time": "2025-04-04T20:20:04.352547279+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 457118630711918595, '"checkpoint_time":' '"2025-04-04' '20:19:50.643",' '"error":' '{' '"time":' '"2025-04-04T20:20:04.352547279+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 457118630711918595, '"checkpoint_time":' '"2025-04-04' '20:19:50.643",' '"error":' '{' '"time":' '"2025-04-04T20:20:04.352547279+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5846.out cli changefeed remove -c changefeed-error check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:20:07 CST 2025] <<<<<< run test case changefeed_reconstruct success! >>>>>> Changefeed remove successfully. ID: changefeed-error CheckpointTs: 457118630711918595 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 31-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:08 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66ab11df-9e50-4a5b-af51-6ee5b49021ea {"id":"66ab11df-9e50-4a5b-af51-6ee5b49021ea","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be481de9 66ab11df-9e50-4a5b-af51-6ee5b49021ea /tidb/cdc/default/default/upstream/7489431659457890943 {"id":7489431659457890943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66ab11df-9e50-4a5b-af51-6ee5b49021ea {"id":"66ab11df-9e50-4a5b-af51-6ee5b49021ea","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be481de9 66ab11df-9e50-4a5b-af51-6ee5b49021ea /tidb/cdc/default/default/upstream/7489431659457890943 {"id":7489431659457890943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66ab11df-9e50-4a5b-af51-6ee5b49021ea {"id":"66ab11df-9e50-4a5b-af51-6ee5b49021ea","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600be481de9 66ab11df-9e50-4a5b-af51-6ee5b49021ea /tidb/cdc/default/default/upstream/7489431659457890943 {"id":7489431659457890943,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7489431659457890943,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2025-04-04T20:20:08.809940814+08:00","start_ts":457118635440734219,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118635440734219,"checkpoint_ts":457118635440734219,"checkpoint_time":"2025-04-04 20:20:08.682"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7489431659457890943, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 457118635440734219, "checkpoint_time": "2025-04-04 20:20:08.682", "error": null }' + echo '{ "upstream_id": 7489431659457890943, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 457118635440734219, "checkpoint_time": "2025-04-04 20:20:08.682", "error": null }' { "upstream_id": 7489431659457890943, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 457118635440734219, "checkpoint_time": "2025-04-04 20:20:08.682", "error": null } ++ echo '{' '"upstream_id":' 7489431659457890943, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 457118635440734219, '"checkpoint_time":' '"2025-04-04' '20:20:08.682",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431659457890943, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 457118635440734219, '"checkpoint_time":' '"2025-04-04' '20:20:08.682",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + set +x check_no_changefeed 127.0.0.1:2379 + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading go1.23.4 (linux/amd64) check diff successfully parse error: Invalid numeric literal at line 1, column 6 run task successfully table ddl_manager.finish_mark not exists for 32-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:20:10 CST 2025] <<<<<< run test case processor_stop_delay success! >>>>>> *************************** 1. row *************************** count(distinct region_id): 40 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 4 20:20:10 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.59285930.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 33-th check, retry later valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri Apr 4 20:20:10 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.54565458.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 34-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:13 GMT < Content-Length: 845 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18d09ca2-a2bd-486e-b2e9-cea3251791b9 {"id":"18d09ca2-a2bd-486e-b2e9-cea3251791b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeae5a 18d09ca2-a2bd-486e-b2e9-cea3251791b9 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18d09ca2-a2bd-486e-b2e9-cea3251791b9 {"id":"18d09ca2-a2bd-486e-b2e9-cea3251791b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeae5a 18d09ca2-a2bd-486e-b2e9-cea3251791b9 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18d09ca2-a2bd-486e-b2e9-cea3251791b9 {"id":"18d09ca2-a2bd-486e-b2e9-cea3251791b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeae5a 18d09ca2-a2bd-486e-b2e9-cea3251791b9 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5980.out cli changefeed create --start-ts=457118627225403393 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7489431545386418571,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:20:14.300848117+08:00","start_ts":457118627225403393,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118627225403393,"checkpoint_ts":457118627225403393,"checkpoint_time":"2025-04-04 20:19:37.343"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading golang.org/x/time v0.7.0 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.9.1 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/goccy/go-json v0.10.2 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 split_and_random_merge scale: 80 check diff successfully go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/pierrec/lz4/v4 v4.1.18 *************************** 1. row *************************** count(distinct region_id): 15 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/golang/protobuf v1.5.4 + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 807 0 --:--:-- --:--:-- --:--:-- 811 + info='{"state":"normal","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010}' + echo '{"state":"normal","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010}' {"state":"normal","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010} ++ echo '{"state":"normal","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.35.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda table ddl_manager.finish_mark not exists for 35-th check, retry later go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.24.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 table ddl_manager.finish_mark not exists for 36-th check, retry later go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 9.61 secs (419379602 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2662 0 --:--:-- --:--:-- --:--:-- 2691 + info='{"state":"warning","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010,"last_warning":{"time":"2025-04-04T20:20:15.879351617+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010,"last_warning":{"time":"2025-04-04T20:20:15.879351617+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010,"last_warning":{"time":"2025-04-04T20:20:15.879351617+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010,"last_warning":{"time":"2025-04-04T20:20:15.879351617+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":457118627592405010,"checkpoint_ts":457118627592405010,"last_warning":{"time":"2025-04-04T20:20:15.879351617+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6034.out cli changefeed remove -c changefeed-error-1 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/go-logr/stdr v1.2.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/qri-io/jsonpointer v0.1.1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 457118627592405010 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/klauspost/cpuid v1.3.1 table ddl_manager.finish_mark not exists for 37-th check, retry later + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd wait process cdc.test exit for 2-th time... *************************** 1. row *************************** count(distinct region_id): 80 table ddl_manager.finish_mark not exists for 38-th check, retry later go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 4 20:20:21 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60936095.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 39-th check, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:24 GMT < Content-Length: 845 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc400919-45d2-487d-a7b3-fabe657cdfb2 {"id":"dc400919-45d2-487d-a7b3-fabe657cdfb2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769222} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeaee4 dc400919-45d2-487d-a7b3-fabe657cdfb2 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc400919-45d2-487d-a7b3-fabe657cdfb2 {"id":"dc400919-45d2-487d-a7b3-fabe657cdfb2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769222} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeaee4 dc400919-45d2-487d-a7b3-fabe657cdfb2 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc400919-45d2-487d-a7b3-fabe657cdfb2 {"id":"dc400919-45d2-487d-a7b3-fabe657cdfb2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769222} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeaee4 dc400919-45d2-487d-a7b3-fabe657cdfb2 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6147.out cli changefeed create --start-ts=457118627225403393 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 check diff failed 4-th time, retry later Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7489431545386418571,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:20:25.29000365+08:00","start_ts":457118627225403393,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118627225403393,"checkpoint_ts":457118627225403393,"checkpoint_time":"2025-04-04 20:19:37.343"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 40-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457118627225403393, "checkpoint_time": "2025-04-04 20:19:37.343", "error": { "time": "2025-04-04T20:20:25.369770268+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457118627225403393, "checkpoint_time": "2025-04-04 20:19:37.343", "error": { "time": "2025-04-04T20:20:25.369770268+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457118627225403393, "checkpoint_time": "2025-04-04 20:19:37.343", "error": { "time": "2025-04-04T20:20:25.369770268+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457118627225403393, '"checkpoint_time":' '"2025-04-04' '20:19:37.343",' '"error":' '{' '"time":' '"2025-04-04T20:20:25.369770268+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457118627225403393, '"checkpoint_time":' '"2025-04-04' '20:19:37.343",' '"error":' '{' '"time":' '"2025-04-04T20:20:25.369770268+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457118627225403393 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6219.out cli changefeed remove -c changefeed-error-2 check diff successfully Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 457118627225403393 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 41-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc) 4028702720 bytes in 8.18 secs (492661427 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431776858168142 TiDB Cluster ID. tikv_gc_leader_uuid 65802fb43040006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:7112, start at 2025-04-04 20:20:27.466690258 +0800 CST m=+2.629505094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:27.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:27.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:27.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431776858168142 TiDB Cluster ID. tikv_gc_leader_uuid 65802fb43040006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:7112, start at 2025-04-04 20:20:27.466690258 +0800 CST m=+2.629505094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:27.474 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:27.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:27.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431779066075561 TiDB Cluster ID. tikv_gc_leader_uuid 65802fb4304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:7181, start at 2025-04-04 20:20:27.481694509 +0800 CST m=+2.580763208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:27.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:27.484 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:27.484 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] sh check diff failed 2-th time, retry later wait process cdc.test exit for 1-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 [Pipeline] sh Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks ci_collation_compatibility PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=0c4ff5ad-8140-49b9-b152-fa4fd418485b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3hjxs GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-3hjxs pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 42-th check, retry later Run cases: availability http_proxies sequence PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=a78f96b2-c57b-4106-9615-6b42db0936c1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-32g4t GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn pingcap_tiflow_periodics_cdc_integration_mysql_test_231-32g4t GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh wait process cdc.test exit for 2-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=401e19b1-6c7d-4d3c-a7c4-43777946e6da BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-qshbg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g pingcap_tiflow_periodics_cdc_integration_mysql_test_231-qshbg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: down_db_require_secure_transport bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=231d0c63-706d-42b4-a750-bc5c4fd7f00c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-j507b GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-j507b pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/down_db_require_secure_transport/run.sh using Sink-Type: mysql... <<================= invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri Apr 4 20:20:30 CST 2025] <<<<<< run all test cases cdc_server_tips success! >>>>>> [Pipeline] sh cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:20:30 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62786280.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=4ad137e8-91bf-4cda-bda9-ad2738524594 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-t65nr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-t65nr pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=f0df452b-2ac5-4482-bb8e-48d40aa90096 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-l1r54 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-l1r54 pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8439.out cli tso query --pd=http://127.0.0.1:2379 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=8a57900e-f5cf-41e7-a081-b1a981aba4d0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-wk88r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-wk88r pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 1-th time, retry later check diff failed 3-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=886e3ed4-fc38-4290-934b-466f86ee9ea3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rc28z GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-rc28z pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=5f57268a-114c-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=e7aa0ecd-6b8d-4180-8425-2ab8dfc23de4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908127571959091204","prowjobid":"5f57268a-114c-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#231 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908127571959091204 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_231-7gw7k GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk pingcap_tiflow_periodics_cdc_integration_mysql_test_231-7gw7k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=231 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 43-th check, retry later + set +x + tso='457118641432297473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118641432297473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... table ddl_manager.finish_mark not exists for 44-th check, retry later check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:33 GMT < Content-Length: 845 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b2617e2-9aa6-4648-b5aa-b8e4f775e043 {"id":"8b2617e2-9aa6-4648-b5aa-b8e4f775e043","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769231} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeaf4d 8b2617e2-9aa6-4648-b5aa-b8e4f775e043 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b2617e2-9aa6-4648-b5aa-b8e4f775e043 {"id":"8b2617e2-9aa6-4648-b5aa-b8e4f775e043","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769231} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeaf4d 8b2617e2-9aa6-4648-b5aa-b8e4f775e043 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b2617e2-9aa6-4648-b5aa-b8e4f775e043 {"id":"8b2617e2-9aa6-4648-b5aa-b8e4f775e043","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769231} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22319600bddeaf4d 8b2617e2-9aa6-4648-b5aa-b8e4f775e043 /tidb/cdc/default/default/upstream/7489431545386418571 {"id":7489431545386418571,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6332.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/down_db_require_secure_transport Starting Upstream PD... ***************** properties ***************** "mysql.user"="root" "threadcount"="2" "mysql.db"="multi_capture_1" "workload"="core" "insertproportion"="0" "scanproportion"="0" "updateproportion"="0" "readallfields"="true" "operationcount"="0" "readproportion"="0" "recordcount"="10" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.host"="127.0.0.1" "dotransactions"="false" ********************************************** Run finished, takes 10.501559ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1768.2, Avg(us): 2008, Min(us): 1183, Max(us): 4827, 95th(us): 5000, 99th(us): 5000 start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7489431545386418571,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:20:34.145283728+08:00","start_ts":457118642088706052,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118642088706052,"checkpoint_ts":457118642088706052,"checkpoint_time":"2025-04-04 20:20:34.042"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431798481400010 TiDB Cluster ID. tikv_gc_leader_uuid 65802fb8ad4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:9291, start at 2025-04-04 20:20:32.096603727 +0800 CST m=+2.394945647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:32.132 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:32.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:32.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431798481400010 TiDB Cluster ID. tikv_gc_leader_uuid 65802fb8ad4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:9291, start at 2025-04-04 20:20:32.096603727 +0800 CST m=+2.394945647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:32.132 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:32.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:32.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431790217498955 TiDB Cluster ID. tikv_gc_leader_uuid 65802fb8a64000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-9-qv8fg, pid:9367, start at 2025-04-04 20:20:32.039697541 +0800 CST m=+2.273932891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:32.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:32.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:32.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ***************** properties ***************** "threadcount"="2" "recordcount"="10" "mysql.user"="root" "dotransactions"="false" "mysql.db"="multi_capture_2" "mysql.host"="127.0.0.1" "readallfields"="true" "requestdistribution"="uniform" "scanproportion"="0" "workload"="core" "updateproportion"="0" "insertproportion"="0" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" ********************************************** Run finished, takes 11.841036ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1575.1, Avg(us): 2247, Min(us): 1209, Max(us): 5423, 95th(us): 6000, 99th(us): 6000 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff failed 5-th time, retry later + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457118642088706052, "checkpoint_time": "2025-04-04 20:20:34.042", "error": { "time": "2025-04-04T20:20:34.325197031+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457118642088706052, "checkpoint_time": "2025-04-04 20:20:34.042", "error": { "time": "2025-04-04T20:20:34.325197031+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457118642088706052, "checkpoint_time": "2025-04-04 20:20:34.042", "error": { "time": "2025-04-04T20:20:34.325197031+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 457118642088706052, '"checkpoint_time":' '"2025-04-04' '20:20:34.042",' '"error":' '{' '"time":' '"2025-04-04T20:20:34.325197031+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... Verifying downstream PD is started... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10589.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 45-th check, retry later ***************** properties ***************** "threadcount"="2" "insertproportion"="0" "mysql.port"="4000" "readproportion"="0" "requestdistribution"="uniform" "workload"="core" "readallfields"="true" "mysql.user"="root" "scanproportion"="0" "mysql.db"="multi_capture_3" "dotransactions"="false" "operationcount"="0" "updateproportion"="0" "recordcount"="10" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.445882ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1786.3, Avg(us): 1976, Min(us): 1131, Max(us): 4693, 95th(us): 5000, 99th(us): 5000 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ***************** properties ***************** "insertproportion"="0" "updateproportion"="0" "readallfields"="true" "mysql.port"="4000" "readproportion"="0" "threadcount"="2" "mysql.db"="multi_capture_4" "requestdistribution"="uniform" "mysql.user"="root" "dotransactions"="false" "operationcount"="0" "workload"="core" "scanproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="10" ********************************************** Run finished, takes 12.034482ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1415.5, Avg(us): 2306, Min(us): 1399, Max(us): 4871, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 20:20:37 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.85688570.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later check diff successfully Verifying downstream PD is started... check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457118642088706052, "checkpoint_time": "2025-04-04 20:20:34.042", "error": { "time": "2025-04-04T20:20:34.325197031+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457118642088706052, "checkpoint_time": "2025-04-04 20:20:34.042", "error": { "time": "2025-04-04T20:20:34.325197031+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457118642088706052, "checkpoint_time": "2025-04-04 20:20:34.042", "error": { "time": "2025-04-04T20:20:34.325197031+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 457118642088706052, '"checkpoint_time":' '"2025-04-04' '20:20:34.042",' '"error":' '{' '"time":' '"2025-04-04T20:20:34.325197031+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + set +x + tso='457118642694520834 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118642694520834 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:20:37 CST 2025] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1062710629.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 46-th check, retry later wait process cdc.test exit for 1-th time... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) config file /tmp/tidb_cdc_test/down_db_require_secure_transport/tidb-config-1743769238361070100.toml contained invalid configuration options: security.socket; check TiDB manual to make sure this option has not been deprecated and removed from your TiDB version if the option does not appear to be a typo Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:20:39 CST 2025] <<<<<< run test case region_merge success! >>>>>> check diff successfully Starting Upstream TiDB... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 47-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:40 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:20:40 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.86308632.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:40 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be161821-0477-405c-998a-42c2ec3f54d7 {"id":"be161821-0477-405c-998a-42c2ec3f54d7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769238} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bec1ace5 be161821-0477-405c-998a-42c2ec3f54d7 /tidb/cdc/default/default/upstream/7489431798481400010 {"id":7489431798481400010,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be161821-0477-405c-998a-42c2ec3f54d7 {"id":"be161821-0477-405c-998a-42c2ec3f54d7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769238} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bec1ace5 be161821-0477-405c-998a-42c2ec3f54d7 /tidb/cdc/default/default/upstream/7489431798481400010 {"id":7489431798481400010,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be161821-0477-405c-998a-42c2ec3f54d7 {"id":"be161821-0477-405c-998a-42c2ec3f54d7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769238} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bec1ace5 be161821-0477-405c-998a-42c2ec3f54d7 /tidb/cdc/default/default/upstream/7489431798481400010 {"id":7489431798481400010,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10681.out cli changefeed create --start-ts=457118642694520834 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 0cb51784-9daf-4d5e-b57f-b660ecdfa43b Info: {"upstream_id":7489431798481400010,"namespace":"default","id":"0cb51784-9daf-4d5e-b57f-b660ecdfa43b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:20:41.406873347+08:00","start_ts":457118642694520834,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118642694520834,"checkpoint_ts":457118642694520834,"checkpoint_time":"2025-04-04 20:20:36.353"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "457118611091488789" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2025/04/04 20:20:37.545 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:20:37.546 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:20:37.546 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:20:37.546 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:20:37.546 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118610907987975] [resolvedTs=457118611091488789] [2025/04/04 20:20:37.547 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:20:37.547 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:20:37.547 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:20:37.547 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.03596ms] [2025/04/04 20:20:37.550 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=3.951848ms] [2025/04/04 20:20:37.551 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:20:37.554 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:20:37.555 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:20:37.556 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:20:37.556 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:20:37.556 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:20:37.556 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:20:37.559 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:20:37.562 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:20:37.562 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/04 20:20:37.562 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118610907987971,\"CommitTs\":457118610907987975,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":116,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:37.563 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:37.563 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118610907987975] [DDL="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [2025/04/04 20:20:37.607 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=44.756629ms] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [2025/04/04 20:20:37.608 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118610921357323,\"CommitTs\":457118610921357326,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":122,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:37.608 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:37.608 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118610921357326] [DDL="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [2025/04/04 20:20:37.636 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=28.060718ms] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [2025/04/04 20:20:37.636 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118610934464543,\"CommitTs\":457118610934464547,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1_1\",\"TableID\":122,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:37.837 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:37.837 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118610934464547] [DDL="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [2025/04/04 20:20:37.868 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=31.446897ms] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [2025/04/04 20:20:37.869 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118610960678922,\"CommitTs\":457118610960678925,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.070 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118610960678925] [DDL="CREATE DATABASE consistent_replicate_ddl1"] [2025/04/04 20:20:38.089 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=19.406919ms] [sql="CREATE DATABASE consistent_replicate_ddl1"] [2025/04/04 20:20:38.089 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118610960678958,\"CommitTs\":457118610973523969,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.090 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:38.090 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118610973523969] [DDL="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/04 20:20:38.107 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=16.919871ms] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/04 20:20:38.107 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118610973524001,\"CommitTs\":457118610973524006,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.107 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:38.107 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118610973524006] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/04 20:20:38.125 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=17.959922ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/04 20:20:38.126 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611000000555,\"CommitTs\":457118611000000562,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.326 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611000000562] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [2025/04/04 20:20:38.376 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=49.239187ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [2025/04/04 20:20:38.376 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611025952781,\"CommitTs\":457118611025952785,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.376 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611025952785] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [2025/04/04 20:20:38.422 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=45.910916ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [2025/04/04 20:20:38.422 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611039322138,\"CommitTs\":457118611039322141,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.422 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611039322141] [DDL="DROP DATABASE consistent_replicate_ddl1"] [2025/04/04 20:20:38.458 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=36.662302ms] [sql="DROP DATABASE consistent_replicate_ddl1"] [2025/04/04 20:20:38.458 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611052167179,\"CommitTs\":457118611052167184,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3\",\"TableID\":120,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.459 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:38.459 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611052167184] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [2025/04/04 20:20:38.479 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=19.82217ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [2025/04/04 20:20:38.479 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611065274391,\"CommitTs\":457118611065274394,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":120,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.479 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:38.479 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611065274394] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [2025/04/04 20:20:38.509 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=29.405917ms] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [2025/04/04 20:20:38.509 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611065274391,\"CommitTs\":457118611065274394,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2_1\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.509 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:20:38.509 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611065274394] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [2025/04/04 20:20:38.538 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=28.58908ms] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [2025/04/04 20:20:38.538 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611078381579,\"CommitTs\":457118611078381583,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":120,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.538 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611078381583] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [2025/04/04 20:20:38.557 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=19.352995ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [2025/04/04 20:20:38.558 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118611091488786,\"CommitTs\":457118611091488789,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"check1\",\"TableID\":137,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:20:38.758 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118611091488789] [DDL="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [2025/04/04 20:20:38.783 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=24.913667ms] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [2025/04/04 20:20:38.783 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457118611091488789] [2025/04/04 20:20:38.783 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457118611091488789] [2025/04/04 20:20:38.884 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}"] [checkpointTs=457118611091488789] [2025/04/04 20:20:38.884 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}"] [checkpointTs=457118611091488789] [2025/04/04 20:20:38.985 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457118611091488789] [2025/04/04 20:20:38.985 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457118611091488789] [2025/04/04 20:20:39.085 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=457118611091488789] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:20:39.085 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Fri Apr 4 20:20:39 CST 2025] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 48-th check, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null } ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457118644094369796, '"checkpoint_time":' '"2025-04-04' '20:20:41.693",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457118644094369796, '"checkpoint_time":' '"2025-04-04' '20:20:41.693",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6473.out cli changefeed pause -c changefeed-initialize-error =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431832264407061 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc0b980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g, pid:1278, start at 2025-04-04 20:20:40.30338794 +0800 CST m=+2.284779498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:40.312 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:40.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:40.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431832264407061 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc0b980003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g, pid:1278, start at 2025-04-04 20:20:40.30338794 +0800 CST m=+2.284779498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:40.312 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:40.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:40.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431839429679723 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc0d40000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g, pid:1351, start at 2025-04-04 20:20:40.416135073 +0800 CST m=+2.312429914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:40.424 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:40.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:40.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431838589519956 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc12a80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:1284, start at 2025-04-04 20:20:40.773021186 +0800 CST m=+2.467775620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:40.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:40.797 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:40.797 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431838589519956 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc12a80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:1284, start at 2025-04-04 20:20:40.773021186 +0800 CST m=+2.467775620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:40.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:40.797 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:40.797 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431837548529497 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc15740006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:1350, start at 2025-04-04 20:20:40.935792388 +0800 CST m=+2.566726297 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:40.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:40.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:40.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431838110521708 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc19880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:1184, start at 2025-04-04 20:20:41.220555572 +0800 CST m=+2.271587606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:41.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:41.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:41.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431838110521708 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc19880012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:1184, start at 2025-04-04 20:20:41.220555572 +0800 CST m=+2.271587606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:41.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:41.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:41.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431837624509735 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc1cc80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:1261, start at 2025-04-04 20:20:41.399812302 +0800 CST m=+2.383783645 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:41.408 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:41.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:41.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 49-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:43 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/capture/34702cb8-a52e-456b-b2df-852c45ec8f3a {"id":"34702cb8-a52e-456b-b2df-852c45ec8f3a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c71 34702cb8-a52e-456b-b2df-852c45ec8f3a /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/capture/34702cb8-a52e-456b-b2df-852c45ec8f3a {"id":"34702cb8-a52e-456b-b2df-852c45ec8f3a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c71 34702cb8-a52e-456b-b2df-852c45ec8f3a /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/capture/34702cb8-a52e-456b-b2df-852c45ec8f3a {"id":"34702cb8-a52e-456b-b2df-852c45ec8f3a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c71 34702cb8-a52e-456b-b2df-852c45ec8f3a /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:20:43 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.86858687.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431833606102796 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc2620000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:1340, start at 2025-04-04 20:20:42.029768361 +0800 CST m=+2.306206566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.039 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.039 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431833606102796 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc2620000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:1340, start at 2025-04-04 20:20:42.029768361 +0800 CST m=+2.306206566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.039 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.039 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431841866908220 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc2798000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:1414, start at 2025-04-04 20:20:42.110149484 +0800 CST m=+2.307719246 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.118 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.086 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.086 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null } ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 457118644094369796, '"checkpoint_time":' '"2025-04-04' '20:20:41.693",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 457118644094369796, '"checkpoint_time":' '"2025-04-04' '20:20:41.693",' '"error":' null '}' ++ jq -r .error.message ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6541.out cli changefeed resume -c changefeed-initialize-error [Fri Apr 4 20:20:44 CST 2025] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26852687.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431838328851428 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc2aa00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:1311, start at 2025-04-04 20:20:42.293052387 +0800 CST m=+2.178089457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.300 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431838328851428 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc2aa00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:1311, start at 2025-04-04 20:20:42.293052387 +0800 CST m=+2.178089457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.300 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.280 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.280 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431846512921723 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc2dac0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:1378, start at 2025-04-04 20:20:42.479944106 +0800 CST m=+2.292076817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.475 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.475 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 4 20:20:44 CST 2025] <<<<<< START cdc server in down_db_require_secure_transport case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.down_db_require_secure_transport.26682670.out server --log-file /tmp/tidb_cdc_test/down_db_require_secure_transport/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/down_db_require_secure_transport/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431842165575368 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc32880002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:1274, start at 2025-04-04 20:20:42.78669113 +0800 CST m=+2.146136620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431842165575368 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc32880002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:1274, start at 2025-04-04 20:20:42.78669113 +0800 CST m=+2.146136620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431846021992277 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc34540006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:1351, start at 2025-04-04 20:20:42.940836513 +0800 CST m=+2.201004050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:42.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:42.951 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:42.951 +0800 All versions after safe point can be accessed. (DO NOT EDIT) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431841347442187 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc38940005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8, pid:1359, start at 2025-04-04 20:20:43.17964627 +0800 CST m=+2.169564090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:43.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:43.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:43.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431841347442187 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc38940005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8, pid:1359, start at 2025-04-04 20:20:43.17964627 +0800 CST m=+2.169564090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:43.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:43.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:43.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431847011184876 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc3afc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8, pid:1432, start at 2025-04-04 20:20:43.363345443 +0800 CST m=+2.298523272 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:43.371 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:43.373 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:43.373 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 4 20:20:45 CST 2025] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26502652.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431843147851979 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc4cc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:1356, start at 2025-04-04 20:20:44.511530174 +0800 CST m=+4.778558999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:44.519 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:44.514 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:44.514 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431843147851979 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc4cc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:1356, start at 2025-04-04 20:20:44.511530174 +0800 CST m=+4.778558999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:44.519 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:44.514 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:44.514 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431842992312666 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc4a78000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:1420, start at 2025-04-04 20:20:44.349432094 +0800 CST m=+4.511427094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:44.437 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:44.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:44.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 4 20:20:46 CST 2025] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.27012703.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + info='{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null }' + echo '{ "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null }' { "upstream_id": 7489431545386418571, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457118644094369796, "checkpoint_time": "2025-04-04 20:20:41.693", "error": null } ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457118644094369796, '"checkpoint_time":' '"2025-04-04' '20:20:41.693",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489431545386418571, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457118644094369796, '"checkpoint_time":' '"2025-04-04' '20:20:41.693",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6607.out cli changefeed remove -c changefeed-initialize-error + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2684.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:46 GMT < Content-Length: 1765 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/capture/34702cb8-a52e-456b-b2df-852c45ec8f3a {"id":"34702cb8-a52e-456b-b2df-852c45ec8f3a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769240} /tidb/cdc/default/__cdc_meta__/capture/f349e6f7-c62a-414c-9b02-b8bd115d6a42 {"id":"f349e6f7-c62a-414c-9b02-b8bd115d6a42","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769243} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c71 34702cb8-a52e-456b-b2df-852c45ec8f3a /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c97 f349e6f7-c62a-414c-9b02-b8bd115d6a42 /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/capture/34702cb8-a52e-456b-b2df-852c45ec8f3a {"id":"34702cb8-a52e-456b-b2df-852c45ec8f3a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769240} /tidb/cdc/default/__cdc_meta__/capture/f349e6f7-c62a-414c-9b02-b8bd115d6a42 {"id":"f349e6f7-c62a-414c-9b02-b8bd115d6a42","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769243} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c71 34702cb8-a52e-456b-b2df-852c45ec8f3a /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c97 f349e6f7-c62a-414c-9b02-b8bd115d6a42 /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab {"id":"085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769237} /tidb/cdc/default/__cdc_meta__/capture/34702cb8-a52e-456b-b2df-852c45ec8f3a {"id":"34702cb8-a52e-456b-b2df-852c45ec8f3a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769240} /tidb/cdc/default/__cdc_meta__/capture/f349e6f7-c62a-414c-9b02-b8bd115d6a42 {"id":"f349e6f7-c62a-414c-9b02-b8bd115d6a42","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769243} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c3c 085cd6bc-0054-4fb8-a1dd-6cae3d3bf9ab /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c71 34702cb8-a52e-456b-b2df-852c45ec8f3a /tidb/cdc/default/__cdc_meta__/owner/22319600beb67c97 f349e6f7-c62a-414c-9b02-b8bd115d6a42 /tidb/cdc/default/default/upstream/7489431776858168142 {"id":7489431776858168142,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8738.out cli changefeed create --start-ts=457118641432297473 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: ea6cc0df-d390-4d24-9cbb-1c2cd0f32378 Info: {"upstream_id":7489431776858168142,"namespace":"default","id":"ea6cc0df-d390-4d24-9cbb-1c2cd0f32378","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:20:46.942725985+08:00","start_ts":457118641432297473,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118641432297473,"checkpoint_ts":457118641432297473,"checkpoint_time":"2025-04-04 20:20:31.538"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Fri Apr 4 20:20:47 CST 2025] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.26292631.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 457118644094369796 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:47 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e74590e7-9c1b-42d4-b004-43c12e0398d0 {"id":"e74590e7-9c1b-42d4-b004-43c12e0398d0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769244} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee9e0d4 e74590e7-9c1b-42d4-b004-43c12e0398d0 /tidb/cdc/default/default/upstream/7489431832264407061 {"id":7489431832264407061,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e74590e7-9c1b-42d4-b004-43c12e0398d0 {"id":"e74590e7-9c1b-42d4-b004-43c12e0398d0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769244} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee9e0d4 e74590e7-9c1b-42d4-b004-43c12e0398d0 /tidb/cdc/default/default/upstream/7489431832264407061 {"id":7489431832264407061,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e74590e7-9c1b-42d4-b004-43c12e0398d0 {"id":"e74590e7-9c1b-42d4-b004-43c12e0398d0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769244} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee9e0d4 e74590e7-9c1b-42d4-b004-43c12e0398d0 /tidb/cdc/default/default/upstream/7489431832264407061 {"id":7489431832264407061,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:20:47 CST 2025] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.26962698.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:47 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8345e2e5-a977-4ad8-aafe-365a5fa6ccb4 {"id":"8345e2e5-a977-4ad8-aafe-365a5fa6ccb4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beeb0dd6 8345e2e5-a977-4ad8-aafe-365a5fa6ccb4 /tidb/cdc/default/default/upstream/7489431838589519956 {"id":7489431838589519956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8345e2e5-a977-4ad8-aafe-365a5fa6ccb4 {"id":"8345e2e5-a977-4ad8-aafe-365a5fa6ccb4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beeb0dd6 8345e2e5-a977-4ad8-aafe-365a5fa6ccb4 /tidb/cdc/default/default/upstream/7489431838589519956 {"id":7489431838589519956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8345e2e5-a977-4ad8-aafe-365a5fa6ccb4 {"id":"8345e2e5-a977-4ad8-aafe-365a5fa6ccb4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beeb0dd6 8345e2e5-a977-4ad8-aafe-365a5fa6ccb4 /tidb/cdc/default/default/upstream/7489431838589519956 {"id":7489431838589519956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 51-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431863456401939 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc72b4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:1283, start at 2025-04-04 20:20:46.917691506 +0800 CST m=+2.674996509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:46.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:46.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:46.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431863456401939 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc72b4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:1283, start at 2025-04-04 20:20:46.917691506 +0800 CST m=+2.674996509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:46.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:46.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:46.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431858633274155 TiDB Cluster ID. tikv_gc_leader_uuid 65802fc73d00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:1358, start at 2025-04-04 20:20:46.976225681 +0800 CST m=+2.673214405 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:46.987 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:46.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:46.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x table multi_capture_1.usertable not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:48 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 {"id":"4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8dd6 4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 /tidb/cdc/default/default/upstream/7489431838110521708 {"id":7489431838110521708,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 {"id":"4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8dd6 4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 /tidb/cdc/default/default/upstream/7489431838110521708 {"id":7489431838110521708,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 {"id":"4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8dd6 4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 /tidb/cdc/default/default/upstream/7489431838110521708 {"id":7489431838110521708,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2725.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 + set +x + tso='457118645573648386 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118645573648386 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:20:48 CST 2025] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.27302732.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 29c2e00a-b60a-4c1b-ade5-c67fb028d048 Info: {"upstream_id":7489431838110521708,"namespace":"default","id":"29c2e00a-b60a-4c1b-ade5-c67fb028d048","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:20:48.935429602+08:00","start_ts":457118645967126532,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118645967126532,"checkpoint_ts":457118645967126532,"checkpoint_time":"2025-04-04 20:20:48.837"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:49 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e912ba0a-cfbe-4dbe-a221-caf7404f66f0 {"id":"e912ba0a-cfbe-4dbe-a221-caf7404f66f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769246} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8aed9 e912ba0a-cfbe-4dbe-a221-caf7404f66f0 /tidb/cdc/default/default/upstream/7489431833606102796 {"id":7489431833606102796,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e912ba0a-cfbe-4dbe-a221-caf7404f66f0 {"id":"e912ba0a-cfbe-4dbe-a221-caf7404f66f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769246} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8aed9 e912ba0a-cfbe-4dbe-a221-caf7404f66f0 /tidb/cdc/default/default/upstream/7489431833606102796 {"id":7489431833606102796,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e912ba0a-cfbe-4dbe-a221-caf7404f66f0 {"id":"e912ba0a-cfbe-4dbe-a221-caf7404f66f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769246} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8aed9 e912ba0a-cfbe-4dbe-a221-caf7404f66f0 /tidb/cdc/default/default/upstream/7489431833606102796 {"id":7489431833606102796,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c4d749f1-6c38-4c1d-b4ef-f5fb0802aeb6 Info: {"upstream_id":7489431833606102796,"namespace":"default","id":"c4d749f1-6c38-4c1d-b4ef-f5fb0802aeb6","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T20:20:49.443024502+08:00","start_ts":457118645272969217,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118645272969217,"checkpoint_ts":457118645272969217,"checkpoint_time":"2025-04-04 20:20:46.189"} table new_ci_collation_test.t1 not exists for 1-th check, retry later check diff failed 4-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.down_db_require_secure_transport.cli.2728.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?ssl-ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem&ssl-key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem&ssl-cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem' --changefeed-id=down-db-require-secure-transport table ddl_manager.finish_mark not exists for 52-th check, retry later + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:20:50 CST 2025] <<<<<< run test case changefeed_error success! >>>>>> table multi_capture_1.usertable exists table partition_table.t2 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:50 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a627037b-ec34-4ef9-81e2-6114a186d9c1 {"id":"a627037b-ec34-4ef9-81e2-6114a186d9c1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769247} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beee1ee2 a627037b-ec34-4ef9-81e2-6114a186d9c1 /tidb/cdc/default/default/upstream/7489431838328851428 {"id":7489431838328851428,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a627037b-ec34-4ef9-81e2-6114a186d9c1 {"id":"a627037b-ec34-4ef9-81e2-6114a186d9c1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769247} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beee1ee2 a627037b-ec34-4ef9-81e2-6114a186d9c1 /tidb/cdc/default/default/upstream/7489431838328851428 {"id":7489431838328851428,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a627037b-ec34-4ef9-81e2-6114a186d9c1 {"id":"a627037b-ec34-4ef9-81e2-6114a186d9c1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769247} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beee1ee2 a627037b-ec34-4ef9-81e2-6114a186d9c1 /tidb/cdc/default/default/upstream/7489431838328851428 {"id":7489431838328851428,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2681.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: down-db-require-secure-transport Info: {"upstream_id":7489431838589519956,"namespace":"default","id":"down-db-require-secure-transport","sink_uri":"mysql://root@127.0.0.1:3306/?ssl-ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026ssl-key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026ssl-cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem","create_time":"2025-04-04T20:20:50.322398664+08:00","start_ts":457118646310273032,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118646310273032,"checkpoint_ts":457118646310273032,"checkpoint_time":"2025-04-04 20:20:50.146"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2679.out cli tso query --pd=http://127.0.0.1:2379 table multi_capture_2.usertable not exists for 1-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:50 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f8ffeea-c97e-4147-9f03-18eb62845415 {"id":"3f8ffeea-c97e-4147-9f03-18eb62845415","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769247} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beedb4d8 3f8ffeea-c97e-4147-9f03-18eb62845415 /tidb/cdc/default/default/upstream/7489431841347442187 {"id":7489431841347442187,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f8ffeea-c97e-4147-9f03-18eb62845415 {"id":"3f8ffeea-c97e-4147-9f03-18eb62845415","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769247} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beedb4d8 3f8ffeea-c97e-4147-9f03-18eb62845415 /tidb/cdc/default/default/upstream/7489431841347442187 {"id":7489431841347442187,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f8ffeea-c97e-4147-9f03-18eb62845415 {"id":"3f8ffeea-c97e-4147-9f03-18eb62845415","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769247} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beedb4d8 3f8ffeea-c97e-4147-9f03-18eb62845415 /tidb/cdc/default/default/upstream/7489431841347442187 {"id":7489431841347442187,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 626baca9-a558-4356-8b2d-078cbe0cac92 Info: {"upstream_id":7489431841347442187,"namespace":"default","id":"626baca9-a558-4356-8b2d-078cbe0cac92","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:20:50.674522699+08:00","start_ts":457118645596454914,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118645596454914,"checkpoint_ts":457118645596454914,"checkpoint_time":"2025-04-04 20:20:47.423"} Create changefeed successfully! ID: 3acf4463-ad87-4b10-94fb-0a270506a939 Info: {"upstream_id":7489431838328851428,"namespace":"default","id":"3acf4463-ad87-4b10-94fb-0a270506a939","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T20:20:50.491619766+08:00","start_ts":457118646384721922,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118646384721922,"checkpoint_ts":457118646384721922,"checkpoint_time":"2025-04-04 20:20:50.430"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 503 Service Unavailable < Accept-Ranges: bytes < Content-Length: 267 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Retry-After: 120 < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18331CE51942C373 < X-Minio-Server-Status: safemode < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 12:20:51 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2742.out cli tso query --pd=http://127.0.0.1:2379 WARNING: Retrying failed request: /?location (503 (XMinioServerNotInitialized): Server not initialized, please try again.) WARNING: Waiting 3 sec... Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide check diff failed 2-th time, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 exists check diff failed 1-th time, retry later check diff failed 5-th time, retry later + set +x table ddl_manager.finish_mark not exists for 53-th check, retry later + set +x [Fri Apr 4 20:20:51 CST 2025] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.27172719.out server --log-file /tmp/tidb_cdc_test/bank/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:51 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83c65581-8094-41a4-956f-d7c745df6565 {"id":"83c65581-8094-41a4-956f-d7c745df6565","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769249} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beec30f0 83c65581-8094-41a4-956f-d7c745df6565 /tidb/cdc/default/default/upstream/7489431842165575368 {"id":7489431842165575368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83c65581-8094-41a4-956f-d7c745df6565 {"id":"83c65581-8094-41a4-956f-d7c745df6565","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769249} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beec30f0 83c65581-8094-41a4-956f-d7c745df6565 /tidb/cdc/default/default/upstream/7489431842165575368 {"id":7489431842165575368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83c65581-8094-41a4-956f-d7c745df6565 {"id":"83c65581-8094-41a4-956f-d7c745df6565","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769249} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beec30f0 83c65581-8094-41a4-956f-d7c745df6565 /tidb/cdc/default/default/upstream/7489431842165575368 {"id":7489431842165575368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2784.out cli changefeed create --start-ts=457118645573648386 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail table partition_table.t2 not exists for 3-th check, retry later table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7489431842165575368,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:20:52.382440053+08:00","start_ts":457118645573648386,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118645573648386,"checkpoint_ts":457118645573648386,"checkpoint_time":"2025-04-04 20:20:47.336"} PASS + set +x + tso='457118646419849217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118646419849217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457118646532308995 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118646532308995 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:20:52 CST 2025] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.27682770.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check diff successfully table ddl_manager.finish_mark not exists for 54-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s check diff successfully [Fri Apr 4 20:20:53 CST 2025] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28642866.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + info='{ "upstream_id": 7489431842165575368, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 457118645573648386, "checkpoint_time": "2025-04-04 20:20:47.336", "error": { "time": "2025-04-04T20:20:52.45755268+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7489431842165575368, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 457118645573648386, "checkpoint_time": "2025-04-04 20:20:47.336", "error": { "time": "2025-04-04T20:20:52.45755268+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7489431842165575368, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 457118645573648386, "checkpoint_time": "2025-04-04 20:20:47.336", "error": { "time": "2025-04-04T20:20:52.45755268+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489431842165575368, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 457118645573648386, '"checkpoint_time":' '"2025-04-04' '20:20:47.336",' '"error":' '{' '"time":' '"2025-04-04T20:20:52.45755268+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489431842165575368, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 457118645573648386, '"checkpoint_time":' '"2025-04-04' '20:20:47.336",' '"error":' '{' '"time":' '"2025-04-04T20:20:52.45755268+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 not exists for 1-th check, retry later run test case test_kill_owner [Fri Apr 4 20:20:54 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.27452747.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_3.usertable exists Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table multi_capture_4.usertable not exists for 1-th check, retry later table partition_table.t2 not exists for 4-th check, retry later Verifying downstream PD is started... table down_db_require_secure_transport.finishmark exists [Fri Apr 4 20:20:54 CST 2025] <<<<<< run test case down_db_require_secure_transport success! >>>>>> wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:54 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58 {"id":"f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119600beee4c12 f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58 /tidb/cdc/default/default/upstream/7489431846512921723 {"id":7489431846512921723,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58 {"id":"f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119600beee4c12 f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58 /tidb/cdc/default/default/upstream/7489431846512921723 {"id":7489431846512921723,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58 {"id":"f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119600beee4c12 f0b3d23a-c0e9-4bcd-b3ff-5e3610d60c58 /tidb/cdc/default/default/upstream/7489431846512921723 {"id":7489431846512921723,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2780.out cli changefeed create --sink-uri=blackhole:// -c changefeed-for-find-finished-ts --server http://127.0.0.1:8400 Create changefeed successfully! ID: changefeed-for-find-finished-ts Info: {"upstream_id":7489431846512921723,"namespace":"default","id":"changefeed-for-find-finished-ts","sink_uri":"blackhole:","create_time":"2025-04-04T20:20:55.334758089+08:00","start_ts":457118647654809604,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118647654809604,"checkpoint_ts":457118647654809604,"checkpoint_time":"2025-04-04 20:20:55.275"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431884324957416 TiDB Cluster ID. tikv_gc_leader_uuid 65802fcd49c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:6466, start at 2025-04-04 20:20:53.16677384 +0800 CST m=+2.196071149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:53.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:53.159 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:53.159 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431884324957416 TiDB Cluster ID. tikv_gc_leader_uuid 65802fcd49c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:6466, start at 2025-04-04 20:20:53.16677384 +0800 CST m=+2.196071149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:53.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:53.159 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:53.159 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431888036160297 TiDB Cluster ID. tikv_gc_leader_uuid 65802fcd70c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:6533, start at 2025-04-04 20:20:53.344982794 +0800 CST m=+2.293391801 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:22:53.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:20:53.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:10:53.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:55 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/302b3669-877c-4a06-a41a-7bf6daaf2bf9 {"id":"302b3669-877c-4a06-a41a-7bf6daaf2bf9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600befa78e2 302b3669-877c-4a06-a41a-7bf6daaf2bf9 /tidb/cdc/default/default/upstream/7489431863456401939 {"id":7489431863456401939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/302b3669-877c-4a06-a41a-7bf6daaf2bf9 {"id":"302b3669-877c-4a06-a41a-7bf6daaf2bf9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600befa78e2 302b3669-877c-4a06-a41a-7bf6daaf2bf9 /tidb/cdc/default/default/upstream/7489431863456401939 {"id":7489431863456401939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/302b3669-877c-4a06-a41a-7bf6daaf2bf9 {"id":"302b3669-877c-4a06-a41a-7bf6daaf2bf9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600befa78e2 302b3669-877c-4a06-a41a-7bf6daaf2bf9 /tidb/cdc/default/default/upstream/7489431863456401939 {"id":7489431863456401939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1398 --:--:-- --:--:-- --:--:-- 1405 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 table ddl_manager.finish_mark not exists for 55-th check, retry later table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:20:56 CST 2025] <<<<<< run test case changefeed_fast_fail success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table multi_capture_4.usertable exists table partition_table.t2 not exists for 5-th check, retry later check diff failed 1-th time, retry later + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading go1.23.4 (linux/amd64) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {UpstreamID:7489431838110521708 Namespace:default ID:29c2e00a-b60a-4c1b-ade5-c67fb028d048 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:48.935429602 +0800 CST StartTs:457118645967126532 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a4f440 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118645980233736} {CheckpointTs:457118647854563333 MinTableBarrierTs:457118647854563333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 {"id":"4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/capture/fea34e50-3b6b-49d5-bc73-2e8b7fda78c2 {"id":"fea34e50-3b6b-49d5-bc73-2e8b7fda78c2","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8dd6 4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8e6e fea34e50-3b6b-49d5-bc73-2e8b7fda78c2 /tidb/cdc/default/default/changefeed/info/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"upstream-id":7489431838110521708,"namespace":"default","changefeed-id":"29c2e00a-b60a-4c1b-ade5-c67fb028d048","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:48.935429602+08:00","start-ts":457118645967126532,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118645980233736} /tidb/cdc/default/default/changefeed/status/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":457118647854563333,"min-table-barrier-ts":457118647854563333,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fea34e50-3b6b-49d5-bc73-2e8b7fda78c2/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431838110521708 {"id":7489431838110521708,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {UpstreamID:7489431838110521708 Namespace:default ID:29c2e00a-b60a-4c1b-ade5-c67fb028d048 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:48.935429602 +0800 CST StartTs:457118645967126532 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a4f440 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118645980233736} {CheckpointTs:457118647854563333 MinTableBarrierTs:457118647854563333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 {"id":"4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/capture/fea34e50-3b6b-49d5-bc73-2e8b7fda78c2 {"id":"fea34e50-3b6b-49d5-bc73-2e8b7fda78c2","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8dd6 4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8e6e fea34e50-3b6b-49d5-bc73-2e8b7fda78c2 /tidb/cdc/default/default/changefeed/info/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"upstream-id":7489431838110521708,"namespace":"default","changefeed-id":"29c2e00a-b60a-4c1b-ade5-c67fb028d048","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:48.935429602+08:00","start-ts":457118645967126532,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118645980233736} /tidb/cdc/default/default/changefeed/status/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":457118647854563333,"min-table-barrier-ts":457118647854563333,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fea34e50-3b6b-49d5-bc73-2e8b7fda78c2/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431838110521708 {"id":7489431838110521708,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {UpstreamID:7489431838110521708 Namespace:default ID:29c2e00a-b60a-4c1b-ade5-c67fb028d048 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:48.935429602 +0800 CST StartTs:457118645967126532 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a4f440 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118645980233736} {CheckpointTs:457118647854563333 MinTableBarrierTs:457118647854563333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 {"id":"4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769245} /tidb/cdc/default/__cdc_meta__/capture/fea34e50-3b6b-49d5-bc73-2e8b7fda78c2 {"id":"fea34e50-3b6b-49d5-bc73-2e8b7fda78c2","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8dd6 4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2 /tidb/cdc/default/__cdc_meta__/owner/22319600beed8e6e fea34e50-3b6b-49d5-bc73-2e8b7fda78c2 /tidb/cdc/default/default/changefeed/info/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"upstream-id":7489431838110521708,"namespace":"default","changefeed-id":"29c2e00a-b60a-4c1b-ade5-c67fb028d048","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:48.935429602+08:00","start-ts":457118645967126532,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118645980233736} /tidb/cdc/default/default/changefeed/status/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":457118647854563333,"min-table-barrier-ts":457118647854563333,"admin-job-type":0} /tidb/cdc/default/default/task/position/4e6a6965-5dfd-4a40-9e6f-cf553b0f78a2/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts"+ grep -q 'etcd info' :0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fea34e50-3b6b-49d5-bc73-2e8b7fda78c2/29c2e00a-b60a-4c1b-ade5-c67fb028d048 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431838110521708 {"id":7489431838110521708,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:20:57 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65196cc3-4b8c-4421-9fe9-945b9666566f {"id":"65196cc3-4b8c-4421-9fe9-945b9666566f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89517 65196cc3-4b8c-4421-9fe9-945b9666566f /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65196cc3-4b8c-4421-9fe9-945b9666566f {"id":"65196cc3-4b8c-4421-9fe9-945b9666566f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89517 65196cc3-4b8c-4421-9fe9-945b9666566f /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65196cc3-4b8c-4421-9fe9-945b9666566f {"id":"65196cc3-4b8c-4421-9fe9-945b9666566f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89517 65196cc3-4b8c-4421-9fe9-945b9666566f /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2795.out cli changefeed create --start-ts=457118646419849217 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff failed 1-th time, retry later Create changefeed successfully! ID: 330a1306-950b-43d9-93e3-10dbf8731a51 Info: {"upstream_id":7489431843147851979,"namespace":"default","id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:20:57.770561931+08:00","start_ts":457118646419849217,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118646419849217,"checkpoint_ts":457118646419849217,"checkpoint_time":"2025-04-04 20:20:50.564"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Fri Apr 4 20:20:57 CST 2025] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.78657867.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 table ddl_manager.finish_mark not exists for 56-th check, retry later table partition_table.t2 not exists for 6-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff successfully ***************** properties ***************** "scanproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "readallfields"="true" "requestdistribution"="uniform" "workload"="core" "mysql.db"="multi_capture_1" "threadcount"="2" "mysql.port"="4000" "insertproportion"="0" "mysql.user"="root" "operationcount"="0" "readproportion"="0" "updateproportion"="0" "recordcount"="20" ********************************************** Run finished, takes 13.319989ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1759.5, Avg(us): 1274, Min(us): 561, Max(us): 2029, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "mysql.db"="multi_capture_2" "threadcount"="2" "mysql.user"="root" "mysql.port"="4000" "operationcount"="0" "workload"="core" "updateproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readproportion"="0" "insertproportion"="0" "dotransactions"="false" "scanproportion"="0" "recordcount"="20" "readallfields"="true" ********************************************** Run finished, takes 21.011644ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1033.8, Avg(us): 2037, Min(us): 699, Max(us): 6934, 95th(us): 7000, 99th(us): 7000 ***************** properties ***************** "updateproportion"="0" "insertproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readproportion"="0" "readallfields"="true" "operationcount"="0" "mysql.db"="multi_capture_3" "workload"="core" "mysql.user"="root" "dotransactions"="false" "threadcount"="2" "recordcount"="20" "requestdistribution"="uniform" ********************************************** Run finished, takes 17.62971ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1263.8, Avg(us): 1365, Min(us): 537, Max(us): 4380, 95th(us): 3000, 99th(us): 5000 ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "operationcount"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" "readproportion"="0" "threadcount"="2" "workload"="core" "recordcount"="20" "requestdistribution"="uniform" "scanproportion"="0" "insertproportion"="0" "dotransactions"="false" "updateproportion"="0" "mysql.db"="multi_capture_4" ********************************************** Run finished, takes 21.777198ms INSERT - Takes(s): 0.0, Count: 20, OPS: 972.5, Avg(us): 2082, Min(us): 491, Max(us): 7478, 95th(us): 6000, 99th(us): 8000 check diff failed 1-th time, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:00 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba94c85f-bb1f-439f-aa84-47d773b92079 {"id":"ba94c85f-bb1f-439f-aa84-47d773b92079","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769257} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf186bd8 ba94c85f-bb1f-439f-aa84-47d773b92079 /tidb/cdc/default/default/upstream/7489431884324957416 {"id":7489431884324957416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba94c85f-bb1f-439f-aa84-47d773b92079 {"id":"ba94c85f-bb1f-439f-aa84-47d773b92079","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769257} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf186bd8 ba94c85f-bb1f-439f-aa84-47d773b92079 /tidb/cdc/default/default/upstream/7489431884324957416 {"id":7489431884324957416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba94c85f-bb1f-439f-aa84-47d773b92079 {"id":"ba94c85f-bb1f-439f-aa84-47d773b92079","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769257} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf186bd8 ba94c85f-bb1f-439f-aa84-47d773b92079 /tidb/cdc/default/default/upstream/7489431884324957416 {"id":7489431884324957416,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.7932.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: dcb72158-9fca-48fe-b8f4-5b390ee125de Info: {"upstream_id":7489431884324957416,"namespace":"default","id":"dcb72158-9fca-48fe-b8f4-5b390ee125de","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:21:00.813921657+08:00","start_ts":457118649092407297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_name*"],"ignore_event":["rename table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118649092407297,"checkpoint_ts":457118649092407297,"checkpoint_time":"2025-04-04 20:21:00.759"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... "is-owner": true, run task successfully check diff failed 3-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 58-th check, retry later go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading google.golang.org/grpc v1.64.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 go: downloading golang.org/x/sys v0.28.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading google.golang.org/protobuf v1.35.2 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda table partition_table.finish_mark not exists for 2-th check, retry later + set +x check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... owner pid: 2750 owner id 65196cc3-4b8c-4421-9fe9-945b9666566f [Fri Apr 4 20:21:02 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.28962898.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431915200289570 TiDB Cluster ID. tikv_gc_leader_uuid 65802fd4680000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:6888, start at 2025-04-04 20:21:00.463421971 +0800 CST m=+2.325208810 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:00.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:00.448 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:00.448 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431915200289570 TiDB Cluster ID. tikv_gc_leader_uuid 65802fd4680000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:6888, start at 2025-04-04 20:21:00.463421971 +0800 CST m=+2.325208810 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:00.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:00.448 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:00.448 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431925351060504 TiDB Cluster ID. tikv_gc_leader_uuid 65802fd4804000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-k-dhj24, pid:6954, start at 2025-04-04 20:21:00.579737618 +0800 CST m=+2.385265854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:00.587 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:00.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:00.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:21:03 CST 2025] <<<<<< run test case multi_capture success! >>>>>> Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 59-th check, retry later check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table partition_table.finish_mark not exists for 3-th check, retry later check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042ae240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118647258710017 MinTableBarrierTs:457118650155401220 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118647258710017, checkpointTs: 457118647258710017, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65196cc3-4b8c-4421-9fe9-945b9666566f {"id":"65196cc3-4b8c-4421-9fe9-945b9666566f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/capture/6dcc135a-9ffd-42ee-a132-f82062a21549 {"id":"6dcc135a-9ffd-42ee-a132-f82062a21549","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769262} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89517 65196cc3-4b8c-4421-9fe9-945b9666566f /tidb/cdc/default/__cdc_meta__/owner/22319600bee89587 6dcc135a-9ffd-42ee-a132-f82062a21549 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118647258710017,"min-table-barrier-ts":457118650155401220,"admin-job-type":0} /tidb/cdc/default/default/task/position/65196cc3-4b8c-4421-9fe9-945b9666566f/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6dcc135a-9ffd-42ee-a132-f82062a21549/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042ae240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118647258710017 MinTableBarrierTs:457118650155401220 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118647258710017, checkpointTs: 457118647258710017, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65196cc3-4b8c-4421-9fe9-945b9666566f {"id":"65196cc3-4b8c-4421-9fe9-945b9666566f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/capture/6dcc135a-9ffd-42ee-a132-f82062a21549 {"id":"6dcc135a-9ffd-42ee-a132-f82062a21549","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769262} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89517 65196cc3-4b8c-4421-9fe9-945b9666566f /tidb/cdc/default/__cdc_meta__/owner/22319600bee89587 6dcc135a-9ffd-42ee-a132-f82062a21549 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118647258710017,"min-table-barrier-ts":457118650155401220,"admin-job-type":0} /tidb/cdc/default/default/task/position/65196cc3-4b8c-4421-9fe9-945b9666566f/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6dcc135a-9ffd-42ee-a132-f82062a21549/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042ae240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118647258710017 MinTableBarrierTs:457118650155401220 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118647258710017, checkpointTs: 457118647258710017, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/65196cc3-4b8c-4421-9fe9-945b9666566f {"id":"65196cc3-4b8c-4421-9fe9-945b9666566f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769254} /tidb/cdc/default/__cdc_meta__/capture/6dcc135a-9ffd-42ee-a132-f82062a21549 {"id":"6dcc135a-9ffd-42ee-a132-f82062a21549","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769262} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89517 65196cc3-4b8c-4421-9fe9-945b9666566f /tidb/cdc/default/__cdc_meta__/owner/22319600bee89587 6dcc135a-9ffd-42ee-a132-f82062a21549 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118647258710017,"min-table-barrier-ts":457118650155401220,"admin-job-type":0} /tidb/cdc/default/default/task/position/65196cc3-4b8c-4421-9fe9-945b9666566f/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6dcc135a-9ffd-42ee-a132-f82062a21549/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "65196cc3-4b8c-4421-9fe9-945b9666566f" | grep -v cluster_id | grep id table ddl_manager.finish_mark not exists for 60-th check, retry later check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "cluster-id": "default" "id": "6dcc135a-9ffd-42ee-a132-f82062a21549", "cluster-id": "default" run task successfully table partition_table.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.8220.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 61-th check, retry later table partition_table.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431946404512511 TiDB Cluster ID. tikv_gc_leader_uuid 65802fda2ac000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:4218, start at 2025-04-04 20:21:06.382610685 +0800 CST m=+2.285522457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:06.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:06.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:06.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431946404512511 TiDB Cluster ID. tikv_gc_leader_uuid 65802fda2ac000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:4218, start at 2025-04-04 20:21:06.382610685 +0800 CST m=+2.285522457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:06.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:06.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:06.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431941142576341 TiDB Cluster ID. tikv_gc_leader_uuid 65802fda4340002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:4302, start at 2025-04-04 20:21:06.447315353 +0800 CST m=+2.275136417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:06.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:06.445 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:06.445 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='457118650701709313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118650701709313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:21:08 CST 2025] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.82568258.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later capture_id: 6dcc135a-9ffd-42ee-a132-f82062a21549 wait process 2750 exit for 1-th time... table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later wait process 2750 exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2750) - No such process wait process 2750 exit for 3-th time... process 2750 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep 6dcc135a-9ffd-42ee-a132-f82062a21549 -A1 | grep '"is-owner": true' table ddl_manager.finish_mark not exists for 62-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table partition_table.finish_mark not exists for 6-th check, retry later check diff failed 3-th time, retry later [Fri Apr 4 20:21:10 CST 2025] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.56885690.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully check diff failed 1-th time, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully test_kill_owner: pass + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d5eee3-f122-4155-8721-2b0387065c9e {"id":"75d5eee3-f122-4155-8721-2b0387065c9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769268} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf3098f8 75d5eee3-f122-4155-8721-2b0387065c9e /tidb/cdc/default/default/upstream/7489431915200289570 {"id":7489431915200289570,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d5eee3-f122-4155-8721-2b0387065c9e {"id":"75d5eee3-f122-4155-8721-2b0387065c9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769268} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf3098f8 75d5eee3-f122-4155-8721-2b0387065c9e /tidb/cdc/default/default/upstream/7489431915200289570 {"id":7489431915200289570,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75d5eee3-f122-4155-8721-2b0387065c9e {"id":"75d5eee3-f122-4155-8721-2b0387065c9e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769268} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf3098f8 75d5eee3-f122-4155-8721-2b0387065c9e /tidb/cdc/default/default/upstream/7489431915200289570 {"id":7489431915200289570,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 9ab984ed-17b9-4aad-8c6b-97c4513d68c0 Info: {"upstream_id":7489431915200289570,"namespace":"default","id":"9ab984ed-17b9-4aad-8c6b-97c4513d68c0","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T20:21:11.538909956+08:00","start_ts":457118650701709313,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118650701709313,"checkpoint_ts":457118650701709313,"checkpoint_time":"2025-04-04 20:21:06.898"} table event_filter.t_alter exists table event_filter.t_name not exists for 1-th check, retry later check diff failed 2-th time, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 63-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... table partition_table.finish_mark not exists for 7-th check, retry later check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431965074731840 TiDB Cluster ID. tikv_gc_leader_uuid 65802fdec140004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:7512, start at 2025-04-04 20:21:11.050668001 +0800 CST m=+2.304675976 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:11.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:11.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:11.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431965074731840 TiDB Cluster ID. tikv_gc_leader_uuid 65802fdec140004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:7512, start at 2025-04-04 20:21:11.050668001 +0800 CST m=+2.304675976 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:11.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:11.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:11.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431969135238630 TiDB Cluster ID. tikv_gc_leader_uuid 65802fded8c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:7591, start at 2025-04-04 20:21:11.143365499 +0800 CST m=+2.330229688 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:11.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:11.139 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:11.139 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:13 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45700c85-1af0-4f48-85fc-1c657c8b4451 {"id":"45700c85-1af0-4f48-85fc-1c657c8b4451","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769270} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf4808d8 45700c85-1af0-4f48-85fc-1c657c8b4451 /tidb/cdc/default/default/upstream/7489431946404512511 {"id":7489431946404512511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45700c85-1af0-4f48-85fc-1c657c8b4451 {"id":"45700c85-1af0-4f48-85fc-1c657c8b4451","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769270} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf4808d8 45700c85-1af0-4f48-85fc-1c657c8b4451 /tidb/cdc/default/default/upstream/7489431946404512511 {"id":7489431946404512511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45700c85-1af0-4f48-85fc-1c657c8b4451 {"id":"45700c85-1af0-4f48-85fc-1c657c8b4451","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769270} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf4808d8 45700c85-1af0-4f48-85fc-1c657c8b4451 /tidb/cdc/default/default/upstream/7489431946404512511 {"id":7489431946404512511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table event_filter.t_name exists table event_filter.t_name1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_hang_up_owner [Fri Apr 4 20:21:13 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30663068.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 64-th check, retry later table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... [2025/04/04 20:21:12.585 +08:00] [INFO] [case.go:450] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:12.586 +08:00] [INFO] [case.go:450] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:21:12.586 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq0] [2025/04/04 20:21:13.103 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.83804ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/04 20:21:13.103 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq1] [2025/04/04 20:21:13.615 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.600958ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/04 20:21:13.616 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq2] [2025/04/04 20:21:14.131 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.406218ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/04 20:21:14.131 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq3] Starting Upstream TiDB... [2025/04/04 20:21:14.644 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.690458ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/04 20:21:14.644 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq4] Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:14 CST 2025] <<<<<< run test case new_ci_collation success! >>>>>> table partition_table.finish_mark not exists for 8-th check, retry later check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8971.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/04 20:21:15.156 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.634794ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/04 20:21:15.156 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts0] check diff failed 5-th time, retry later wait process cdc.test exit for 1-th time... table event_filter.t_name1 exists table event_filter.t_name2 not exists for 1-th check, retry later [2025/04/04 20:21:15.668 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.580368ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/04 20:21:15.668 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts1] table ddl_manager.finish_mark not exists for 65-th check, retry later table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:21:16.181 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.581436ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/04 20:21:16.181 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts2] check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:16 CST 2025] <<<<<< run test case owner_resign success! >>>>>> table partition_table.finish_mark not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:21:16.694 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.534506ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/04 20:21:16.694 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts3] + set +x + tso='457118652942254081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118652942254081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:21:16 CST 2025] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.90169018.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042baab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118650941833222 MinTableBarrierTs:457118653038985225 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3363c97c-3683-4e9c-9d1b-51d85d600191 {"id":"3363c97c-3683-4e9c-9d1b-51d85d600191","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8960c 3363c97c-3683-4e9c-9d1b-51d85d600191 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118650941833222,"min-table-barrier-ts":457118653038985225,"admin-job-type":0} /tidb/cdc/default/default/task/position/3363c97c-3683-4e9c-9d1b-51d85d600191/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042baab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118650941833222 MinTableBarrierTs:457118653038985225 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3363c97c-3683-4e9c-9d1b-51d85d600191 {"id":"3363c97c-3683-4e9c-9d1b-51d85d600191","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8960c 3363c97c-3683-4e9c-9d1b-51d85d600191 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118650941833222,"min-table-barrier-ts":457118653038985225,"admin-job-type":0} /tidb/cdc/default/default/task/position/3363c97c-3683-4e9c-9d1b-51d85d600191/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042baab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118650941833222 MinTableBarrierTs:457118653038985225 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118650941833222, checkpointTs: 457118650941833222, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3363c97c-3683-4e9c-9d1b-51d85d600191 {"id":"3363c97c-3683-4e9c-9d1b-51d85d600191","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8960c 3363c97c-3683-4e9c-9d1b-51d85d600191 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118650941833222,"min-table-barrier-ts":457118653038985225,"admin-job-type":0} /tid+ grep -q 'etcd info' b/cdc/default/default/task/position/3363c97c-3683-4e9c-9d1b-51d85d600191/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff failed 6-th time, retry later [2025/04/04 20:21:17.206 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.306168ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/04 20:21:17.206 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts4] table event_filter.t_name2 exists table event_filter.t_name3 not exists for 1-th check, retry later [2025/04/04 20:21:17.718 +08:00] [INFO] [case.go:370] ["insert 100 takes 4.258195ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] table ddl_manager.finish_mark not exists for 66-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table consistent_replicate_gbk.check1 not exists for 2-th check, retry later table partition_table.finish_mark exists check diff successfully Verifying downstream PD is started... "is-owner": true, run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431991508846459 TiDB Cluster ID. tikv_gc_leader_uuid 65802fe47000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:3730, start at 2025-04-04 20:21:16.870664223 +0800 CST m=+2.404433861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:16.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:16.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:16.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431991508846459 TiDB Cluster ID. tikv_gc_leader_uuid 65802fe47000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:3730, start at 2025-04-04 20:21:16.870664223 +0800 CST m=+2.404433861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:16.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:16.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:16.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431994001912333 TiDB Cluster ID. tikv_gc_leader_uuid 65802fe4984000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:3817, start at 2025-04-04 20:21:17.047468422 +0800 CST m=+2.496700479 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:17.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:17.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:17.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 7-th time, retry later wait process cdc.test exit for 2-th time... table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431987594966910 TiDB Cluster ID. tikv_gc_leader_uuid 65802fe5ae00021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:3879, start at 2025-04-04 20:21:18.142847419 +0800 CST m=+2.333977607 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:18.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:18.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:18.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431987594966910 TiDB Cluster ID. tikv_gc_leader_uuid 65802fe5ae00021 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:3879, start at 2025-04-04 20:21:18.142847419 +0800 CST m=+2.333977607 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:18.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:18.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:18.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489431995650546590 TiDB Cluster ID. tikv_gc_leader_uuid 65802fe5cf00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:3958, start at 2025-04-04 20:21:18.279550718 +0800 CST m=+2.407031842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:18.287 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:18.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:18.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... table event_filter.t_name3 exists [2025/04/04 20:21:20.225 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:20 CST 2025] <<<<<< run test case partition_table success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:19 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/456a234e-031e-4d63-be37-07851186ca38 {"id":"456a234e-031e-4d63-be37-07851186ca38","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769277} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf5e17e3 456a234e-031e-4d63-be37-07851186ca38 /tidb/cdc/default/default/upstream/7489431965074731840 {"id":7489431965074731840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/456a234e-031e-4d63-be37-07851186ca38 {"id":"456a234e-031e-4d63-be37-07851186ca38","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769277} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf5e17e3 456a234e-031e-4d63-be37-07851186ca38 /tidb/cdc/default/default/upstream/7489431965074731840 {"id":7489431965074731840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/456a234e-031e-4d63-be37-07851186ca38 {"id":"456a234e-031e-4d63-be37-07851186ca38","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769277} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf5e17e3 456a234e-031e-4d63-be37-07851186ca38 /tidb/cdc/default/default/upstream/7489431965074731840 {"id":7489431965074731840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9075.out cli changefeed create --start-ts=457118652942254081 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } owner pid: 3071 owner id 3363c97c-3683-4e9c-9d1b-51d85d600191 [Fri Apr 4 20:21:20 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.31793181.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_gbk.check1 exists check diff successfully table ddl_manager.finish_mark not exists for 67-th check, retry later Create changefeed successfully! ID: 1fb369c8-77a9-4204-b418-2a69f87f05f4 Info: {"upstream_id":7489431965074731840,"namespace":"default","id":"1fb369c8-77a9-4204-b418-2a69f87f05f4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:21:20.337516318+08:00","start_ts":457118652942254081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118652942254081,"checkpoint_ts":457118652942254081,"checkpoint_time":"2025-04-04 20:21:15.445"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... table common_1.recover_and_insert not exists for 2-th check, retry later [Fri Apr 4 20:21:20 CST 2025] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.51365138.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... check diff failed 8-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:21 CST 2025] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.58865888.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table ddl_manager.finish_mark not exists for 68-th check, retry later check diff failed 1-th time, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later table event_filter.finish_mark exists check diff failed 1-th time, retry later [2025/04/04 20:21:22.231 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table common_1.recover_and_insert not exists for 3-th check, retry later check diff failed 9-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00101c750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118654873993232 MinTableBarrierTs:457118654873993232 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3363c97c-3683-4e9c-9d1b-51d85d600191 {"id":"3363c97c-3683-4e9c-9d1b-51d85d600191","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769274} /tidb/cdc/default/__cdc_meta__/capture/5155e2e9-718c-418b-9e41-2ab5024c53f2 {"id":"5155e2e9-718c-418b-9e41-2ab5024c53f2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8960c 3363c97c-3683-4e9c-9d1b-51d85d600191 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89677 5155e2e9-718c-418b-9e41-2ab5024c53f2 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118654873993232,"min-table-barrier-ts":457118654873993232,"admin-job-type":0} /tidb/cdc/default/default/task/position/3363c97c-3683-4e9c-9d1b-51d85d600191/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5155e2e9-718c-418b-9e41-2ab5024c53f2/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00101c750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118654873993232 MinTableBarrierTs:457118654873993232 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3363c97c-3683-4e9c-9d1b-51d85d600191 {"id":"3363c97c-3683-4e9c-9d1b-51d85d600191","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769274} /tidb/cdc/default/__cdc_meta__/capture/5155e2e9-718c-418b-9e41-2ab5024c53f2 {"id":"5155e2e9-718c-418b-9e41-2ab5024c53f2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8960c 3363c97c-3683-4e9c-9d1b-51d85d600191 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89677 5155e2e9-718c-418b-9e41-2ab5024c53f2 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118654873993232,"min-table-barrier-ts":457118654873993232,"admin-job-type":0} /tidb/cdc/default/default/task/position/3363c97c-3683-4e9c-9d1b-51d85d600191/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5155e2e9-718c-418b-9e41-2ab5024c53f2/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00101c750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118654873993232 MinTableBarrierTs:457118654873993232 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3363c97c-3683-4e9c-9d1b-51d85d600191 {"id":"3363c97c-3683-4e9c-9d1b-51d85d600191","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769274} /tidb/cdc/default/__cdc_meta__/capture/5155e2e9-718c-418b-9e41-2ab5024c53f2 {"id":"5155e2e9-718c-418b-9e41-2ab5024c53f2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769280} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8960c 3363c97c-3683-4e9c-9d1b-51d85d600191 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89677 5155e2e9-718c-418b-9e41-2ab5024c53f2 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118654873993232,"min-table-barrier-ts":457118654873993232,"admin-job-type":0} /tidb/cdc/default/default/task/position/3363c97c-3683-4e9c-9d1b-51d85d600191/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5155e2e9-718c-418b-9e41-2ab5024c53f2/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "3363c97c-3683-4e9c-9d1b-51d85d600191" | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:23 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff2b1ff1-f1ed-499c-97ed-864f98ec0065 {"id":"ff2b1ff1-f1ed-499c-97ed-864f98ec0065","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf7661d7 ff2b1ff1-f1ed-499c-97ed-864f98ec0065 /tidb/cdc/default/default/upstream/7489431991508846459 {"id":7489431991508846459,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff2b1ff1-f1ed-499c-97ed-864f98ec0065 {"id":"ff2b1ff1-f1ed-499c-97ed-864f98ec0065","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf7661d7 ff2b1ff1-f1ed-499c-97ed-864f98ec0065 /tidb/cdc/default/default/upstream/7489431991508846459 {"id":7489431991508846459,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff2b1ff1-f1ed-499c-97ed-864f98ec0065 {"id":"ff2b1ff1-f1ed-499c-97ed-864f98ec0065","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf7661d7 ff2b1ff1-f1ed-499c-97ed-864f98ec0065 /tidb/cdc/default/default/upstream/7489431991508846459 {"id":7489431991508846459,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:21:23 CST 2025] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.51935195.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 69-th check, retry later check diff failed 2-th time, retry later table ddl_sequence.finish_mark not exists for 2-th check, retry later check diff successfully [Fri Apr 4 20:21:24 CST 2025] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.52965298.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:21:24.237 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... build mysql_test successfully run all mysql test cases time="2025-04-04T20:21:21+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2025-04-04T20:21:21+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001ac000 {{} {} 0} {0 0} [0xc00031e000] {[]} 1 0xc0001b0000 false map[0xc00031e000:map[0xc00031e000:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" ./t/alter_table.test: ok! 64 test cases passed, take time 1.536424535 s time="2025-04-04T20:21:23+08:00" level=info msg="run test [alter_table] ok" time="2025-04-04T20:21:23+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000308380 {{} {} 0} {0 0} [0xc0001bf580] {[]} 1 0xc000300930 false map[0xc0001bf580:map[0xc0001bf580:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" "cluster-id": "default" "id": "5155e2e9-718c-418b-9e41-2ab5024c53f2", "cluster-id": "default" run task successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {UpstreamID:7489431946404512511 Namespace:default ID:202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:21:13.712365619 +0800 CST StartTs:457118652470919172 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001877dd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118652484026371} {CheckpointTs:457118652536455169 MinTableBarrierTs:457118655236800514 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118652536455169, checkpointTs: 457118652536455169, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118652536455169, checkpointTs: 457118652536455169, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/707d8fab-6cc0-4b46-bc03-cf2e0355c26e {"id":"707d8fab-6cc0-4b46-bc03-cf2e0355c26e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf480982 707d8fab-6cc0-4b46-bc03-cf2e0355c26e /tidb/cdc/default/default/changefeed/info/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"upstream-id":7489431946404512511,"namespace":"default","changefeed-id":"202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:21:13.712365619+08:00","start-ts":457118652470919172,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118652484026371} /tidb/cdc/default/default/changefeed/status/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"checkpoint-ts":457118652536455169,"min-table-barrier-ts":457118655236800514,"admin-job-type":0} /tidb/cdc/default/default/task/position/707d8fab-6cc0-4b46-bc03-cf2e0355c26e/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431946404512511 {"id":7489431946404512511,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {UpstreamID:7489431946404512511 Namespace:default ID:202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:21:13.712365619 +0800 CST StartTs:457118652470919172 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001877dd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118652484026371} {CheckpointTs:457118652536455169 MinTableBarrierTs:457118655236800514 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118652536455169, checkpointTs: 457118652536455169, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118652536455169, checkpointTs: 457118652536455169, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/707d8fab-6cc0-4b46-bc03-cf2e0355c26e {"id":"707d8fab-6cc0-4b46-bc03-cf2e0355c26e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf480982 707d8fab-6cc0-4b46-bc03-cf2e0355c26e /tidb/cdc/default/default/changefeed/info/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"upstream-id":7489431946404512511,"namespace":"default","changefeed-id":"202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:21:13.712365619+08:00","start-ts":457118652470919172,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118652484026371} /tidb/cdc/default/default/changefeed/status/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"checkpoint-ts":457118652536455169,"min-table-barrier-ts":457118655236800514,"admin-job-type":0} /tidb/cdc/default/default/task/position/707d8fab-6cc0-4b46-bc03-cf2e0355c26e/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431946404512511 {"id":7489431946404512511,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {UpstreamID:7489431946404512511 Namespace:default ID:202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:21:13.712365619 +0800 CST StartTs:457118652470919172 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001877dd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118652484026371} {CheckpointTs:457118652536455169 MinTableBarrierTs:457118655236800514 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118652536455169, checkpointTs: 457118652536455169, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118652536455169, checkpointTs: 457118652536455169, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/707d8fab-6cc0-4b46-bc03-cf2e0355c26e {"id":"707d8fab-6cc0-4b46-bc03-cf2e0355c26e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf480982 707d8fab-6cc0-4b46-bc03-cf2e0355c26e /tidb/cdc/default/default/changefeed/info/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"upstream-id":7489431946404512511,"namespace":"default","changefeed-id":"202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:21:13.712365619+08:00","start-ts":457118652470919172,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118652484026371} /tidb/cdc/default/default/changefeed/status/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"checkpoint-ts":457118652536455169,"min-table-barrier-ts":457118655236800514,"admin-job-type":0} /tidb/cdc/default/default/task/position/707d8fab-6cc0-4b46-bc03-cf2e0355c26e/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b {"checkpoint-t+ grep -q 'etcd info' s":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431946404512511 {"id":7489431946404512511,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432019748370811 TiDB Cluster ID. tikv_gc_leader_uuid 65802feb5f80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:10086, start at 2025-04-04 20:21:23.974992998 +0800 CST m=+2.625698588 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:23.983 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:23.966 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:23.966 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432019748370811 TiDB Cluster ID. tikv_gc_leader_uuid 65802feb5f80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:10086, start at 2025-04-04 20:21:23.974992998 +0800 CST m=+2.625698588 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:23.983 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:23.966 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:23.966 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432020908103380 TiDB Cluster ID. tikv_gc_leader_uuid 65802feb6a80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:10155, start at 2025-04-04 20:21:24.015318931 +0800 CST m=+2.596439722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:24.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:24.061 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:24.061 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 2-th time... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 10-th time, retry later table ddl_manager.finish_mark not exists for 70-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:25 CST 2025] <<<<<< run test case event_filter success! >>>>>> check diff failed 3-th time, retry later capture_id: 5155e2e9-718c-418b-9e41-2ab5024c53f2 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '5155e2e9-718c-418b-9e41-2ab5024c53f2' 5155e2e9-718c-418b-9e41-2ab5024c53f2 run task successfully test_hang_up_owner: pass [2025/04/04 20:21:26.243 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_sequence.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... table common_1.finish_mark not exists for 2-th check, retry later ./t/bigint.test: ok! 138 test cases passed, take time 4.094085777 s time="2025-04-04T20:21:27+08:00" level=info msg="run test [bigint] ok" time="2025-04-04T20:21:27+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000308200 {{} {} 0} {0 0} [0xc000270000] {[]} 1 0xc00002ccb0 false map[0xc000270000:map[0xc000270000:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" ./t/composite_index.test: ok! 12 test cases passed, take time 0.089698853 s time="2025-04-04T20:21:27+08:00" level=info msg="run test [composite_index] ok" time="2025-04-04T20:21:27+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0002c00c0 {{} {} 0} {0 0} [0xc000371e80] {[]} 1 0xc000336380 false map[0xc000371e80:map[0xc000371e80:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9f2433-919e-4c3d-b392-9c4de8774469 {"id":"8b9f2433-919e-4c3d-b392-9c4de8774469","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769284} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119600bf769301 8b9f2433-919e-4c3d-b392-9c4de8774469 /tidb/cdc/default/default/upstream/7489431994001912333 {"id":7489431994001912333,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9f2433-919e-4c3d-b392-9c4de8774469 {"id":"8b9f2433-919e-4c3d-b392-9c4de8774469","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769284} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119600bf769301 8b9f2433-919e-4c3d-b392-9c4de8774469 /tidb/cdc/default/default/upstream/7489431994001912333 {"id":7489431994001912333,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b9f2433-919e-4c3d-b392-9c4de8774469 {"id":"8b9f2433-919e-4c3d-b392-9c4de8774469","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769284} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119600bf769301 8b9f2433-919e-4c3d-b392-9c4de8774469 /tidb/cdc/default/default/upstream/7489431994001912333 {"id":7489431994001912333,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.5246.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml wait process cdc.test exit for 2-th time... ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 0.088137082 s time="2025-04-04T20:21:27+08:00" level=info msg="run test [concurrent_ddl] ok" time="2025-04-04T20:21:27+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003082b8 {{} {} 0} {0 0} [0xc000634000] {[]} 1 0xc00002d1f0 false map[0xc000634000:map[0xc000634000:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489431991508846459,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2025-04-04T20:21:27.474620104+08:00","start_ts":457118656066748420,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118656066748420,"checkpoint_ts":457118656066748420,"checkpoint_time":"2025-04-04 20:21:27.364"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3384b47-f9d3-45f5-8bc8-d7def2e5959b {"id":"c3384b47-f9d3-45f5-8bc8-d7def2e5959b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769284} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf759ef0 c3384b47-f9d3-45f5-8bc8-d7def2e5959b /tidb/cdc/default/default/upstream/7489431987594966910 {"id":7489431987594966910,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3384b47-f9d3-45f5-8bc8-d7def2e5959b {"id":"c3384b47-f9d3-45f5-8bc8-d7def2e5959b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769284} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf759ef0 c3384b47-f9d3-45f5-8bc8-d7def2e5959b /tidb/cdc/default/default/upstream/7489431987594966910 {"id":7489431987594966910,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3384b47-f9d3-45f5-8bc8-d7def2e5959b {"id":"c3384b47-f9d3-45f5-8bc8-d7def2e5959b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769284} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf759ef0 c3384b47-f9d3-45f5-8bc8-d7def2e5959b /tidb/cdc/default/default/upstream/7489431987594966910 {"id":7489431987594966910,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5352.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489431987594966910,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2025-04-04T20:21:27.700848943+08:00","start_ts":457118656133595140,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118656133595140,"checkpoint_ts":457118656133595140,"checkpoint_time":"2025-04-04 20:21:27.619"} PASS check diff failed 11-th time, retry later ./t/date_formats.test: ok! 99 test cases passed, take time 0.42384706 s time="2025-04-04T20:21:27+08:00" level=info msg="run test [date_formats] ok" time="2025-04-04T20:21:27+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003082d0 {{} {} 0} {0 0} [0xc00026e000] {[]} 1 0xc000696c40 false map[0xc00026e000:map[0xc00026e000:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11434.out cli tso query --pd=http://127.0.0.1:2379 coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 71-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found ./t/drop.test: ok! 48 test cases passed, take time 0.542983326 s time="2025-04-04T20:21:28+08:00" level=info msg="run test [drop] ok" time="2025-04-04T20:21:28+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001ac220 {{} {} 0} {0 0} [0xc0001b8780] {[]} 1 0xc00002caf0 false map[0xc0001b8780:map[0xc0001b8780:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" [2025/04/04 20:21:28.248 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_owner [Fri Apr 4 20:21:28 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33433345.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ./t/mysql_replace.test: ok! 16 test cases passed, take time 0.077454903 s time="2025-04-04T20:21:28+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end table test.finish_mark not exists for 1-th check, retry later check diff failed 4-th time, retry later table ddl_sequence.finish_mark not exists for 4-th check, retry later table common_1.finish_mark not exists for 3-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.5276.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml + set +x Create changefeed successfully! ID: test-2 Info: {"upstream_id":7489431994001912333,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2025-04-04T20:21:29.327887438+08:00","start_ts":457118656551714820,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118656551714820,"checkpoint_ts":457118656551714820,"checkpoint_time":"2025-04-04 20:21:29.214"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 12-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457118656250773507 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118656250773507 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118656250773507 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 20:21:29 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1146711469.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_with_exists/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:21:29 CST 2025] <<<<<< run test case ddl_with_exists success! >>>>>> check_redo_resolved_ts 202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b 457118656717651973 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b/meta global resolved ts 457118655380717569 not forward to 457118656717651973 run task failed 1-th time, retry later [2025/04/04 20:21:30.254 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table ddl_manager.finish_mark not exists for 72-th check, retry later check diff successfully table test.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 5-th check, retry later table common_1.finish_mark not exists for 4-th check, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043a1200 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118655660425231 MinTableBarrierTs:457118656984514564 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/540dc869-bd4a-4579-b689-a64bbde33142 {"id":"540dc869-bd4a-4579-b689-a64bbde33142","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee896db 540dc869-bd4a-4579-b689-a64bbde33142 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118655660425231,"min-table-barrier-ts":457118656984514564,"admin-job-type":0} /tidb/cdc/default/default/task/position/540dc869-bd4a-4579-b689-a64bbde33142/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043a1200 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118655660425231 MinTableBarrierTs:457118656984514564 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/540dc869-bd4a-4579-b689-a64bbde33142 {"id":"540dc869-bd4a-4579-b689-a64bbde33142","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee896db 540dc869-bd4a-4579-b689-a64bbde33142 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118655660425231,"min-table-barrier-ts":457118656984514564,"admin-job-type":0} /tidb/cdc/default/default/task/position/540dc869-bd4a-4579-b689-a64bbde33142/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043a1200 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118655660425231 MinTableBarrierTs:457118656984514564 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118655660425231, checkpointTs: 457118655660425231, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/540dc869-bd4a-4579-b689-a64bbde33142 {"id":"540dc869-bd4a-4579-b689-a64bbde33142","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee896db 540dc869-bd4a-4579-b689-a64bbde33142 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118655660425231,"min-table-barrier-ts":457118656984514564,"admin-job-type":0} /tid+ grep -q 'etcd info' b/cdc/default/default/task/position/540dc869-bd4a-4579-b689-a64bbde33142/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_redo_resolved_ts 202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b 457118656717651973 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b/meta global resolved ts 457118655380717569 not forward to 457118656717651973 run task failed 2-th time, retry later check diff failed 13-th time, retry later Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 73-th check, retry later table ddl_sequence.finish_mark not exists for 6-th check, retry later [2025/04/04 20:21:32.260 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] table test.finish_mark not exists for 3-th check, retry later "is-owner": true, run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:32 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c0f0640-76e0-4c56-ad61-b3ff9366781f {"id":"1c0f0640-76e0-4c56-ad61-b3ff9366781f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf9126ea 1c0f0640-76e0-4c56-ad61-b3ff9366781f /tidb/cdc/default/default/upstream/7489432019748370811 {"id":7489432019748370811,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c0f0640-76e0-4c56-ad61-b3ff9366781f {"id":"1c0f0640-76e0-4c56-ad61-b3ff9366781f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf9126ea 1c0f0640-76e0-4c56-ad61-b3ff9366781f /tidb/cdc/default/default/upstream/7489432019748370811 {"id":7489432019748370811,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c0f0640-76e0-4c56-ad61-b3ff9366781f {"id":"1c0f0640-76e0-4c56-ad61-b3ff9366781f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bf9126ea 1c0f0640-76e0-4c56-ad61-b3ff9366781f /tidb/cdc/default/default/upstream/7489432019748370811 {"id":7489432019748370811,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118656250773507 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11530.out cli changefeed create --start-ts=457118656250773507 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489432019748370811,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:21:33.036797429+08:00","start_ts":457118656250773507,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118656250773507,"checkpoint_ts":457118656250773507,"checkpoint_time":"2025-04-04 20:21:28.066"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table common_1.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) wait process cdc.test exit for 1-th time... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 14-th time, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 74-th check, retry later [Pipeline] // node table ddl_sequence.finish_mark exists [Pipeline] } [Pipeline] // podTemplate owner pid: 3348 owner id 540dc869-bd4a-4579-b689-a64bbde33142 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '540dc869-bd4a-4579-b689-a64bbde33142' /tidb/cdc/default/__cdc_meta__/owner/22319600bee896db run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Pipeline] } + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5410.out cli changefeed pause -c test check diff successfully [Pipeline] // withEnv [Pipeline] } check diff failed 1-th time, retry later [2025/04/04 20:21:34.266 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2025/04/04 20:21:34.271 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:34.273 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5119 0 --:--:-- --:--:-- --:--:-- 5139 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 20:21:28.066","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 20:21:34.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:21:28.066","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:21:34.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:21:28.066","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:21:34.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2025-04-04 20:21:28.066' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:21:28.066","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:21:34.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:21:28.066","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:21:34.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2025-04-04 20:21:34' + echo 'sink_checkpoint_ts is 2025-04-04' 20:21:28.066 sink_checkpoint_ts is 2025-04-04 20:21:28.066 ++ date -d '2025-04-04 20:21:28.066' +%s + checkpoint_timestamp=1743769288 ++ date -d '2025-04-04 20:21:34' +%s + current_timestamp=1743769294 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:34 CST 2025] <<<<<< run test case common_1 success! >>>>>> [Pipeline] // stage PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] } table test.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:33 CST 2025] <<<<<< run test case many_pk_or_uk success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore wait process cdc.test exit for 2-th time... Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 15-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5444.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm table ddl_manager.finish_mark not exists for 75-th check, retry later [Fri Apr 4 20:21:35 CST 2025] <<<<<< run test case ddl_sequence success! >>>>>> check_redo_resolved_ts 202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b 457118656717651973 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/202a0647-c4cb-4bd6-bb93-bfb00fd0ad4b/meta run task successfully "is-owner": true, run task successfully test_expire_owner pass check diff failed 2-th time, retry later [2025/04/04 20:21:36.280 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2025/04/04 20:21:36.285 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:36.287 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118657708032008 : [2025/04/04 20:21:34.429 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=134] [jobID=135] [query="CREATE TABLE IF NOT EXISTS `finishmark` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118657708032008] [2025/04/04 20:21:36.292 +08:00] [INFO] [case.go:503] ["all tables synced"] [endTs=457118657708032008] wait process cdc.test exit for 1-th time... Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} Update changefeed config successfully! ID: test Info: {"upstream_id":7489431987594966910,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2025-04-04T20:21:27.700848943+08:00","start_ts":457118656133595140,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457118656932872204,"checkpoint_time":"2025-04-04 20:21:30.668"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... table test.t1 exists + sleep 5 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Fri Apr 4 20:21:37 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35233525.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2025/04/04 20:21:37.813 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:21:37.814 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:21:37.814 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:21:37.815 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/04 20:21:37.818 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:21:37.818 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118655380717569] [resolvedTs=457118657595834371] [2025/04/04 20:21:37.819 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/04 20:21:37.819 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/04 20:21:37.819 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:21:37.819 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:21:37.819 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:21:37.822 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=3.824411ms] [2025/04/04 20:21:37.823 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=5.273489ms] [2025/04/04 20:21:37.824 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:21:37.826 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:21:37.827 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:21:37.828 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:21:37.828 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:21:37.828 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:21:37.828 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:21:37.832 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:21:37.834 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:21:37.834 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/04 20:21:37.834 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118655367872531,\"CommitTs\":457118655380717569,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_gbk\",\"Table\":\"GBKTABLE2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:21:37.834 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118655380717569] [DDL="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [2025/04/04 20:21:37.836 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457118655367872531] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432072870352881 TiDB Cluster ID. tikv_gc_leader_uuid 65802ff6ba0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:4140, start at 2025-04-04 20:21:35.610834316 +0800 CST m=+2.203879965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:35.618 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:35.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:35.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432072870352881 TiDB Cluster ID. tikv_gc_leader_uuid 65802ff6ba0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:4140, start at 2025-04-04 20:21:35.610834316 +0800 CST m=+2.203879965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:35.618 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:35.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:35.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432067156956931 TiDB Cluster ID. tikv_gc_leader_uuid 65802ff6d180006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:4216, start at 2025-04-04 20:21:35.702447874 +0800 CST m=+2.241660038 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:35.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:35.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:35.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 76-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5469.out cli changefeed resume -c test [2025/04/04 20:21:38.036 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457118657595834371] [2025/04/04 20:21:38.036 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457118657595834371] [2025/04/04 20:21:38.136 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=457118657595834371] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/04 20:21:38.137 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:21:38.137 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:21:38.136 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully check diff failed 16-th time, retry later check diff successfully [Fri Apr 4 20:21:38 CST 2025] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT check diff failed 3-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 6-th check, retry later [Fri Apr 4 20:21:39 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.54785480.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432072911615997 TiDB Cluster ID. tikv_gc_leader_uuid 65802ff8dfc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:4145, start at 2025-04-04 20:21:37.79339197 +0800 CST m=+2.221418910 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:37.802 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:37.791 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:37.791 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432072911615997 TiDB Cluster ID. tikv_gc_leader_uuid 65802ff8dfc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:4145, start at 2025-04-04 20:21:37.79339197 +0800 CST m=+2.221418910 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:37.802 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:37.791 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:37.791 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432081821585892 TiDB Cluster ID. tikv_gc_leader_uuid 65802ff8f5c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:4191, start at 2025-04-04 20:21:37.918366267 +0800 CST m=+2.290930273 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:37.925 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:37.924 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:37.924 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 17-th time, retry later table ddl_manager.finish_mark not exists for 77-th check, retry later table test.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00127fb90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118658032828420 MinTableBarrierTs:457118659343548421 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118658032828420,"min-table-barrier-ts":457118659343548421,"admin-job-type":0} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00127fb90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118658032828420 MinTableBarrierTs:457118659343548421 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118658032828420,"min-table-barrier-ts":457118659343548421,"admin-job-type":0} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00127fb90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118658032828420 MinTableBarrierTs:457118659343548421 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118658032828420, checkpointTs: 457118658032828420, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118658032828420,"min-table-barrier-ts":457118659343548421,"admin-job-type":0} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2025/04/04 20:21:40.154 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark1000] [2025/04/04 20:21:40.154 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark1000] check diff failed 4-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5516.out cli changefeed pause -c test [Fri Apr 4 20:21:41 CST 2025] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.54225424.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 18-th time, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4731 0 --:--:-- --:--:-- --:--:-- 4673 100 243 100 243 0 0 4722 0 --:--:-- --:--:-- --:--:-- 4673 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:21:40.416","puller_resolved_ts":"2025-04-04 20:21:34.666","last_synced_ts":"2025-04-04 20:21:35.166","now_ts":"2025-04-04 20:21:42.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:21:40.416","puller_resolved_ts":"2025-04-04' '20:21:34.666","last_synced_ts":"2025-04-04' '20:21:35.166","now_ts":"2025-04-04' '20:21:42.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:21:40.416","puller_resolved_ts":"2025-04-04' '20:21:34.666","last_synced_ts":"2025-04-04' '20:21:35.166","now_ts":"2025-04-04' '20:21:42.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table ddl_manager.finish_mark not exists for 78-th check, retry later "is-owner": true, run task successfully [2025/04/04 20:21:42.155 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2025/04/04 20:21:42.163 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark1000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:42.165 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff successfully table test.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:42 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ae962a-44a9-42c1-927c-9cbec9708d85 {"id":"48ae962a-44a9-42c1-927c-9cbec9708d85","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769300} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfbc65d7 48ae962a-44a9-42c1-927c-9cbec9708d85 /tidb/cdc/default/default/upstream/7489432072870352881 {"id":7489432072870352881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ae962a-44a9-42c1-927c-9cbec9708d85 {"id":"48ae962a-44a9-42c1-927c-9cbec9708d85","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769300} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfbc65d7 48ae962a-44a9-42c1-927c-9cbec9708d85 /tidb/cdc/default/default/upstream/7489432072870352881 {"id":7489432072870352881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ae962a-44a9-42c1-927c-9cbec9708d85 {"id":"48ae962a-44a9-42c1-927c-9cbec9708d85","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769300} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfbc65d7 48ae962a-44a9-42c1-927c-9cbec9708d85 /tidb/cdc/default/default/upstream/7489432072870352881 {"id":7489432072870352881,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5533.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 8f26a449-d037-47e9-b9a0-fd1d00eb5611 Info: {"upstream_id":7489432072870352881,"namespace":"default","id":"8f26a449-d037-47e9-b9a0-fd1d00eb5611","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:21:43.321282061+08:00","start_ts":457118660215177222,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118660215177222,"checkpoint_ts":457118660215177222,"checkpoint_time":"2025-04-04 20:21:43.189"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5546.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} owner pid: 3528 owner id e7edd5a4-1206-4211-aa89-d5590b88df69 [Fri Apr 4 20:21:43 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.36493651.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Update changefeed config successfully! ID: test Info: {"upstream_id":7489431987594966910,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2025-04-04T20:21:27.700848943+08:00","start_ts":457118656133595140,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457118657063944209,"checkpoint_time":"2025-04-04 20:21:31.168"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 79-th check, retry later [2025/04/04 20:21:44.267 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2025/04/04 20:21:44.274 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark1000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:44.275 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118659792338953 : [2025/04/04 20:21:42.479 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=136] [jobID=137] [query="CREATE TABLE IF NOT EXISTS `finishmark1000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118659792338953] [2025/04/04 20:21:44.373 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark1000] [2025/04/04 20:21:44.373 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118659792338953] [2025/04/04 20:21:44.377 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:44.456 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:21:44.463 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:44.468 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff failed 19-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } + set +x [2025/04/04 20:21:44.668 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark2000] [2025/04/04 20:21:44.668 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark2000] [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache table test.finish_mark not exists for 9-th check, retry later [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:44 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9638294-0dd0-4488-8656-0d24e7e9c1c1 {"id":"f9638294-0dd0-4488-8656-0d24e7e9c1c1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfc2ced8 f9638294-0dd0-4488-8656-0d24e7e9c1c1 /tidb/cdc/default/default/upstream/7489432072911615997 {"id":7489432072911615997,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9638294-0dd0-4488-8656-0d24e7e9c1c1 {"id":"f9638294-0dd0-4488-8656-0d24e7e9c1c1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfc2ced8 f9638294-0dd0-4488-8656-0d24e7e9c1c1 /tidb/cdc/default/default/upstream/7489432072911615997 {"id":7489432072911615997,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9638294-0dd0-4488-8656-0d24e7e9c1c1 {"id":"f9638294-0dd0-4488-8656-0d24e7e9c1c1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfc2ced8 f9638294-0dd0-4488-8656-0d24e7e9c1c1 /tidb/cdc/default/default/upstream/7489432072911615997 {"id":7489432072911615997,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5576.out cli changefeed resume -c test [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 80-th check, retry later check diff failed 20-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.finish_mark not exists for 10-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003851c20 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118660916674565 MinTableBarrierTs:457118660916674566 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7715594-ffac-45c6-af8d-dd60072af6d9 {"id":"c7715594-ffac-45c6-af8d-dd60072af6d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769303} /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89794 c7715594-ffac-45c6-af8d-dd60072af6d9 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118660916674565,"min-table-barrier-ts":457118660916674566,"admin-job-type":0} /tidb/cdc/default/default/task/position/c7715594-ffac-45c6-af8d-dd60072af6d9/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003851c20 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118660916674565 MinTableBarrierTs:457118660916674566 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7715594-ffac-45c6-af8d-dd60072af6d9 {"id":"c7715594-ffac-45c6-af8d-dd60072af6d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769303} /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89794 c7715594-ffac-45c6-af8d-dd60072af6d9 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118660916674565,"min-table-barrier-ts":457118660916674566,"admin-job-type":0} /tidb/cdc/default/default/task/position/c7715594-ffac-45c6-af8d-dd60072af6d9/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003851c20 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118660916674565 MinTableBarrierTs:457118660916674566 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7715594-ffac-45c6-af8d-dd60072af6d9 {"id":"c7715594-ffac-45c6-af8d-dd60072af6d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769303} /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89794 c7715594-ffac-45c6-af8d-dd60072af6d9 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118660916674565,"min-table-barrier-ts":457118660916674566,"admin-job-type":0} /tidb/cdc/default/default/task/position/c7715594-ffac-45c6-af8d-dd60072af6d9/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 + grep -q 'etcd info' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "e7edd5a4-1206-4211-aa89-d5590b88df69" | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 2-th check, retry later [2025/04/04 20:21:46.753 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2025/04/04 20:21:46.763 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark2000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:46.765 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118660782456843 : [2025/04/04 20:21:45.479 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=138] [jobID=139] [query="CREATE TABLE IF NOT EXISTS `finishmark2000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118660782456843] [2025/04/04 20:21:46.963 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark2000] [2025/04/04 20:21:46.963 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118660782456843] [2025/04/04 20:21:46.965 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:46.968 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:21:46.974 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:47.059 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + set +x table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later "cluster-id": "default" "id": "c7715594-ffac-45c6-af8d-dd60072af6d9", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 81-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 11-th check, retry later check diff failed 2-th time, retry later [2025/04/04 20:21:34.505 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 11"] [2025/04/04 20:21:39.507 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 23"] check diff successfully wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432118507295449 TiDB Cluster ID. tikv_gc_leader_uuid 658030024b00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9258, start at 2025-04-04 20:21:47.447314816 +0800 CST m=+2.293463141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark not exists for 3-th check, retry later [2025/04/04 20:21:49.461 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark3000] [2025/04/04 20:21:49.461 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark3000] wait process cdc.test exit for 2-th time... table batch_update_to_no_batch.recover_and_insert not exists for 2-th check, retry later capture_id: c7715594-ffac-45c6-af8d-dd60072af6d9 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:50 CST 2025] <<<<<< run test case clustered_index success! >>>>>> table ddl_manager.finish_mark not exists for 82-th check, retry later table test.finish_mark not exists for 12-th check, retry later check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.10586.out cli tso query --pd=http://127.0.0.1:2379 table batch_add_table.finish_mark not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:21:51.461 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2025/04/04 20:21:51.468 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark3000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:51.469 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118662093176847 : [2025/04/04 20:21:50.479 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=140] [jobID=141] [query="CREATE TABLE IF NOT EXISTS `finishmark3000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118662093176847] [2025/04/04 20:21:51.666 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark3000] [2025/04/04 20:21:51.666 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118662093176847] [2025/04/04 20:21:51.670 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:51.673 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:21:51.765 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:51.854 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 83-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 13-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 [Fri Apr 4 20:21:52 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.37743776.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later + set +x + tso='457118662480101377 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118662480101377 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118662480101377 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri Apr 4 20:21:53 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1062110623.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 5-th check, retry later table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 84-th check, retry later [2025/04/04 20:21:54.154 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark4000] [2025/04/04 20:21:54.154 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark4000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table test.finish_mark not exists for 14-th check, retry later table batch_add_table.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:55 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003dd27e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118661702844423 MinTableBarrierTs:457118661702844424 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5065c892-9d92-43d1-b82c-dbe256236e50 {"id":"5065c892-9d92-43d1-b82c-dbe256236e50","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769312} /tidb/cdc/default/__cdc_meta__/capture/c7715594-ffac-45c6-af8d-dd60072af6d9 {"id":"c7715594-ffac-45c6-af8d-dd60072af6d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769303} /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89794 c7715594-ffac-45c6-af8d-dd60072af6d9 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89811 5065c892-9d92-43d1-b82c-dbe256236e50 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118661702844423,"min-table-barrier-ts":457118661702844424,"admin-job-type":0} /tidb/cdc/default/default/task/position/5065c892-9d92-43d1-b82c-dbe256236e50/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c7715594-ffac-45c6-af8d-dd60072af6d9/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003dd27e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118661702844423 MinTableBarrierTs:457118661702844424 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5065c892-9d92-43d1-b82c-dbe256236e50 {"id":"5065c892-9d92-43d1-b82c-dbe256236e50","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769312} /tidb/cdc/default/__cdc_meta__/capture/c7715594-ffac-45c6-af8d-dd60072af6d9 {"id":"c7715594-ffac-45c6-af8d-dd60072af6d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769303} /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89794 c7715594-ffac-45c6-af8d-dd60072af6d9 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89811 5065c892-9d92-43d1-b82c-dbe256236e50 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} + grep -q 'failed to get info:' /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118661702844423,"min-table-barrier-ts":457118661702844424,"admin-job-type":0} /tidb/cdc/default/default/task/position/5065c892-9d92-43d1-b82c-dbe256236e50/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c7715594-ffac-45c6-af8d-dd60072af6d9/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003dd27e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118661702844423 MinTableBarrierTs:457118661702844424 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5065c892-9d92-43d1-b82c-dbe256236e50 {"id":"5065c892-9d92-43d1-b82c-dbe256236e50","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769312} /tidb/cdc/default/__cdc_meta__/capture/c7715594-ffac-45c6-af8d-dd60072af6d9 {"id":"c7715594-ffac-45c6-af8d-dd60072af6d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769303} /tidb/cdc/default/__cdc_meta__/capture/e7edd5a4-1206-4211-aa89-d5590b88df69 {"id":"e7edd5a4-1206-4211-aa89-d5590b88df69","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769297} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89741 e7edd5a4-1206-4211-aa89-d5590b88df69 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89794 c7715594-ffac-45c6-af8d-dd60072af6d9 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89811 5065c892-9d92-43d1-b82c-dbe256236e50 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118661702844423,"min-table-barrier-ts":457118661702844424,"admin-job-type":0} /tidb/cdc/default/default/task/position/5065c892-9d92-43d1-b82c-dbe256236e50/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/c7715594-ffac-45c6-af8d-dd60072af6d9/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e7edd5a4-1206-4211-aa89-d5590b88df69/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 85-th check, retry later [2025/04/04 20:21:56.154 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2025/04/04 20:21:56.161 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark4000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:56.163 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:21:56 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78d6f51c-c6a9-4350-93fa-00a88cb63434 {"id":"78d6f51c-c6a9-4350-93fa-00a88cb63434","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769313} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfea11e7 78d6f51c-c6a9-4350-93fa-00a88cb63434 /tidb/cdc/default/default/upstream/7489432119985441098 {"id":7489432119985441098,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78d6f51c-c6a9-4350-93fa-00a88cb63434 {"id":"78d6f51c-c6a9-4350-93fa-00a88cb63434","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769313} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfea11e7 78d6f51c-c6a9-4350-93fa-00a88cb63434 /tidb/cdc/default/default/upstream/7489432119985441098 {"id":7489432119985441098,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78d6f51c-c6a9-4350-93fa-00a88cb63434 {"id":"78d6f51c-c6a9-4350-93fa-00a88cb63434","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769313} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfea11e7 78d6f51c-c6a9-4350-93fa-00a88cb63434 /tidb/cdc/default/default/upstream/7489432119985441098 {"id":7489432119985441098,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118662480101377 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.10675.out cli changefeed create --start-ts=457118662480101377 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489432119985441098,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:21:56.706440274+08:00","start_ts":457118662480101377,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118662480101377,"checkpoint_ts":457118662480101377,"checkpoint_time":"2025-04-04 20:21:51.829"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 15-th check, retry later table batch_add_table.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432156166148306 TiDB Cluster ID. tikv_gc_leader_uuid 65803009e04000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:21387, start at 2025-04-04 20:21:55.232363937 +0800 CST m=+2.329862351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:55.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:55.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:55.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432156166148306 TiDB Cluster ID. tikv_gc_leader_uuid 65803009e04000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:21387, start at 2025-04-04 20:21:55.232363937 +0800 CST m=+2.329862351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:55.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:55.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:55.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432153710247113 TiDB Cluster ID. tikv_gc_leader_uuid 6580300a0500007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:21458, start at 2025-04-04 20:21:55.360591647 +0800 CST m=+2.396818397 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:55.367 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:55.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:55.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_update_to_no_batch.finish_mark exists cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' table ddl_manager.finish_mark not exists for 86-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:21:58.175 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark5000] [2025/04/04 20:21:58.175 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark5000] check diff successfully + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 run task failed 2-th time, retry later [2025/04/04 20:21:58.456 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2025/04/04 20:21:58.463 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark4000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:21:58.465 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118663482540046 : [2025/04/04 20:21:56.579 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=142] [jobID=143] [query="CREATE TABLE IF NOT EXISTS `finishmark4000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118663482540046] [2025/04/04 20:21:58.755 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark4000] [2025/04/04 20:21:58.756 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118663482540046] [2025/04/04 20:21:58.759 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:21:58.764 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:21:58.773 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432163815947250 TiDB Cluster ID. tikv_gc_leader_uuid 6580300ba300012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:10068, start at 2025-04-04 20:21:57.04299024 +0800 CST m=+2.384309859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:57.097 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:57.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:57.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432163815947250 TiDB Cluster ID. tikv_gc_leader_uuid 6580300ba300012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:10068, start at 2025-04-04 20:21:57.04299024 +0800 CST m=+2.384309859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:57.097 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:57.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:57.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432155104422002 TiDB Cluster ID. tikv_gc_leader_uuid 6580300bbc0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-26zqv, pid:10131, start at 2025-04-04 20:21:57.14574502 +0800 CST m=+2.412365941 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:57.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:57.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:57.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + '[' mysql == mysql ']' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 10 table test.finish_mark not exists for 16-th check, retry later [2025/04/04 20:21:58.858 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] wait process cdc.test exit for 2-th time... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:21:59 CST 2025] <<<<<< run test case batch_update_to_no_batch success! >>>>>> table batch_add_table.finish_mark not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 87-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:22:00.253 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2025/04/04 20:22:00.260 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark5000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:00.262 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.22880.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 17-th check, retry later check diff failed 2-th time, retry later table batch_add_table.finish_mark not exists for 9-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11527.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432179197209587 TiDB Cluster ID. tikv_gc_leader_uuid 6580300e9b4003e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:7087, start at 2025-04-04 20:22:00.075429324 +0800 CST m=+2.330029165 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:00.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:00.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:00.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432179197209587 TiDB Cluster ID. tikv_gc_leader_uuid 6580300e9b4003e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:7087, start at 2025-04-04 20:22:00.075429324 +0800 CST m=+2.330029165 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:00.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:00.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:00.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432177608851569 TiDB Cluster ID. tikv_gc_leader_uuid 6580300eb38000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:7163, start at 2025-04-04 20:22:00.168300111 +0800 CST m=+2.363898426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:00.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:00.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:00.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' table ddl_manager.finish_mark not exists for 88-th check, retry later + set +x + tso='457118664818950145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118664818950145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:22:02 CST 2025] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2292322925.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 18-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 [2025/04/04 20:22:02.564 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2025/04/04 20:22:02.569 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark5000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:02.571 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] [2025/04/04 20:22:02.955 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark6000] [2025/04/04 20:22:02.955 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark6000] found first match line: 457118664531116047 : [2025/04/04 20:22:00.579 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=144] [jobID=145] [query="CREATE TABLE IF NOT EXISTS `finishmark5000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118664531116047] [2025/04/04 20:22:02.956 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark5000] [2025/04/04 20:22:02.956 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118664531116047] [2025/04/04 20:22:02.959 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] + set +x + tso='457118664977285121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118664977285121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:22:02 CST 2025] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1156611568.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later Verifying upstream PD is started... [2025/04/04 20:22:02.964 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:03.055 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:03.063 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table batch_add_table.finish_mark not exists for 10-th check, retry later "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark exists check diff successfully [Fri Apr 4 20:22:04 CST 2025] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.85688570.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... [2025/04/04 20:22:04.956 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2025/04/04 20:22:04.966 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark6000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:04.967 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table test.finish_mark not exists for 19-th check, retry later wait process cdc.test exit for 2-th time... table batch_add_table.finish_mark not exists for 11-th check, retry later found first match line: 457118665632120848 : [2025/04/04 20:22:04.653 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=146] [jobID=147] [query="CREATE TABLE IF NOT EXISTS `finishmark6000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118665632120848] [2025/04/04 20:22:05.263 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark6000] [2025/04/04 20:22:05.263 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118665632120848] [2025/04/04 20:22:05.266 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:05.269 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:05.281 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:05.353 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:05 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb839525-5858-4577-9942-6453fc4ca937 {"id":"eb839525-5858-4577-9942-6453fc4ca937","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c006aafd eb839525-5858-4577-9942-6453fc4ca937 /tidb/cdc/default/default/upstream/7489432156166148306 {"id":7489432156166148306,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb839525-5858-4577-9942-6453fc4ca937 {"id":"eb839525-5858-4577-9942-6453fc4ca937","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c006aafd eb839525-5858-4577-9942-6453fc4ca937 /tidb/cdc/default/default/upstream/7489432156166148306 {"id":7489432156166148306,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb839525-5858-4577-9942-6453fc4ca937 {"id":"eb839525-5858-4577-9942-6453fc4ca937","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c006aafd eb839525-5858-4577-9942-6453fc4ca937 /tidb/cdc/default/default/upstream/7489432156166148306 {"id":7489432156166148306,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:05 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cfe9c9c-4793-4709-a655-4342a0b428f7 {"id":"9cfe9c9c-4793-4709-a655-4342a0b428f7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c00f78e0 9cfe9c9c-4793-4709-a655-4342a0b428f7 /tidb/cdc/default/default/upstream/7489432163815947250 {"id":7489432163815947250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cfe9c9c-4793-4709-a655-4342a0b428f7 {"id":"9cfe9c9c-4793-4709-a655-4342a0b428f7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c00f78e0 9cfe9c9c-4793-4709-a655-4342a0b428f7 /tidb/cdc/default/default/upstream/7489432163815947250 {"id":7489432163815947250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cfe9c9c-4793-4709-a655-4342a0b428f7 {"id":"9cfe9c9c-4793-4709-a655-4342a0b428f7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c00f78e0 9cfe9c9c-4793-4709-a655-4342a0b428f7 /tidb/cdc/default/default/upstream/7489432163815947250 {"id":7489432163815947250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11614.out cli changefeed create --start-ts=457118664977285121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 4-th time... Create changefeed successfully! ID: 3f77bca3-455d-4c9f-ade2-2a37f486b21a Info: {"upstream_id":7489432163815947250,"namespace":"default","id":"3f77bca3-455d-4c9f-ade2-2a37f486b21a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:22:06.222537135+08:00","start_ts":457118664977285121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118664977285121,"checkpoint_ts":457118664977285121,"checkpoint_time":"2025-04-04 20:22:01.355"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:22:06.448 +08:00] [INFO] [pd_service_discovery.go:1025] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2025/04/04 20:22:06.449 +08:00] [INFO] [pd_service_discovery.go:499] ["[pd] init cluster id"] [cluster-id=7489431423981950614] [2025/04/04 20:22:06.449 +08:00] [INFO] [client.go:532] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 20:22:06.450 +08:00] [INFO] [tso_client.go:296] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2025/04/04 20:22:06.451 +08:00] [INFO] [client.go:538] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 20:22:06.451 +08:00] [INFO] [tso_dispatcher.go:140] ["[tso] start tso deadline watcher"] [dc-location=global] [2025/04/04 20:22:06.451 +08:00] [INFO] [tso_client.go:132] ["[tso] start tso dispatcher check loop"] [2025/04/04 20:22:06.451 +08:00] [INFO] [tso_dispatcher.go:198] ["[tso] tso dispatcher created"] [dc-location=global] [2025/04/04 20:22:06.451 +08:00] [INFO] [tso_dispatcher.go:476] ["[tso] start tso connection contexts updater"] [dc-location=global] [2025/04/04 20:22:06.451 +08:00] [INFO] [tso_dispatcher.go:723] ["[tso] switching tso rpc concurrency"] [old=0] [new=1] [2025/04/04 20:22:06.452 +08:00] [INFO] [pd_service_discovery.go:1025] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2025/04/04 20:22:06.452 +08:00] [INFO] [pd_service_discovery.go:499] ["[pd] init cluster id"] [cluster-id=7489431423981950614] [2025/04/04 20:22:06.452 +08:00] [INFO] [client.go:532] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 20:22:06.452 +08:00] [INFO] [tso_client.go:296] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2025/04/04 20:22:06.452 +08:00] [INFO] [client.go:538] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 20:22:06.452 +08:00] [INFO] [tso_dispatcher.go:140] ["[tso] start tso deadline watcher"] [dc-location=global] [2025/04/04 20:22:06.452 +08:00] [INFO] [tso_client.go:132] ["[tso] start tso dispatcher check loop"] [2025/04/04 20:22:06.452 +08:00] [INFO] [tso_dispatcher.go:198] ["[tso] tso dispatcher created"] [dc-location=global] [2025/04/04 20:22:06.453 +08:00] [INFO] [tso_dispatcher.go:723] ["[tso] switching tso rpc concurrency"] [old=0] [new=1] [2025/04/04 20:22:06.453 +08:00] [INFO] [tso_dispatcher.go:476] ["[tso] start tso connection contexts updater"] [dc-location=global] [2025/04/04 20:22:06.453 +08:00] [INFO] [tikv_driver.go:201] ["using API V1."] [2025/04/04 20:22:06.454 +08:00] [INFO] [main.go:180] ["genLock started"] [2025/04/04 20:22:06.455 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=4] [addr=127.0.0.1:20162] [from=unresolved] [to=resolved] [liveness-state=reachable] lease 22319600c006aafd revoked wait process cdc.test exit for 5-th time... table test.finish_mark not exists for 20-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:07 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/457a6543-c7c8-4874-bf56-5f0bc1155a6a {"id":"457a6543-c7c8-4874-bf56-5f0bc1155a6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769324} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c01995d8 457a6543-c7c8-4874-bf56-5f0bc1155a6a /tidb/cdc/default/default/upstream/7489432179197209587 {"id":7489432179197209587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/457a6543-c7c8-4874-bf56-5f0bc1155a6a {"id":"457a6543-c7c8-4874-bf56-5f0bc1155a6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769324} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c01995d8 457a6543-c7c8-4874-bf56-5f0bc1155a6a /tidb/cdc/default/default/upstream/7489432179197209587 {"id":7489432179197209587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/457a6543-c7c8-4874-bf56-5f0bc1155a6a {"id":"457a6543-c7c8-4874-bf56-5f0bc1155a6a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769324} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c01995d8 457a6543-c7c8-4874-bf56-5f0bc1155a6a /tidb/cdc/default/default/upstream/7489432179197209587 {"id":7489432179197209587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully wait process cdc.test exit for 6-th time... table capture_session_done_during_task.t exists check diff failed 1-th time, retry later + set +x table resourcecontrol.finish_mark not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 12-th check, retry later [2025/04/04 20:22:07.853 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark7000] [2025/04/04 20:22:07.854 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark7000] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 7-th time... wait process cdc.test exit for 2-th time... nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 8-th time... process cdc.test already exit [Fri Apr 4 20:22:08 CST 2025] <<<<<< run test case ddl_manager success! >>>>>> [2025/04/04 20:22:08.453 +08:00] [INFO] [pd.go:441] ["adaptive update ts interval state transition"] [configuredInterval=2s] [prevAdaptiveUpdateInterval=2s] [newAdaptiveUpdateInterval=2s] [requiredStaleness=0s] [prevState=unknown(0)] [newState=normal] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "requestdistribution"="uniform" "scanproportion"="0" "dotransactions"="false" "fieldcount"="3" "operationcount"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "workload"="core" "recordcount"="50000" "mysql.db"="consistent_replicate_nfs" "readallfields"="true" "mysql.user"="root" "insertproportion"="0" "threadcount"="10" "mysql.port"="4000" ********************************************** cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:08 CST 2025] <<<<<< run test case changefeed_pause_resume success! >>>>>> + run_sql 'SELECT * from test.t1' 127.0.0.1 3306 + check_contains 'b: 123456' + check_contains 'b: aaaaa' + check_contains 'c: ab\\\\c' + check_contains 'c: ab\\c' + stop_tidb_cluster table test.finish_mark not exists for 21-th check, retry later table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 exists table bdr_mode.finish_mark exists table bdr_mode.finish_mark not exists for 1-th check, retry later table bdr_mode.finish_mark not exists for 2-th check, retry later table bdr_mode.finish_mark exists =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table batch_add_table.finish_mark not exists for 13-th check, retry later check diff successfully empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass check diff failed 2-th time, retry later table resourcecontrol.finish_mark exists check diff successfully [2025/04/04 20:22:09.854 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2025/04/04 20:22:09.954 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark7000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:09.956 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118666837983247 : [2025/04/04 20:22:08.579 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=148] [jobID=149] [query="CREATE TABLE IF NOT EXISTS `finishmark7000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118666837983247] [2025/04/04 20:22:10.364 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark7000] [2025/04/04 20:22:10.364 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118666837983247] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... [2025/04/04 20:22:10.454 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:10.458 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:10.473 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:10.556 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 22-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:11 CST 2025] <<<<<< run test case bdr_mode success! >>>>>> wait process cdc.test exit for 2-th time... table batch_add_table.finish_mark not exists for 14-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:11 CST 2025] <<<<<< run test case resourcecontrol success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_retryable_error [Fri Apr 4 20:22:11 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.40274029.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully check diff failed 1-th time, retry later [2025/04/04 20:22:12.073 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark8000] [2025/04/04 20:22:12.073 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark8000] table test.finish_mark not exists for 23-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432223020059214 TiDB Cluster ID. tikv_gc_leader_uuid 658030191340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8, pid:4405, start at 2025-04-04 20:22:10.773658263 +0800 CST m=+2.311307673 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:10.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:10.765 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:10.765 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432223020059214 TiDB Cluster ID. tikv_gc_leader_uuid 658030191340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8, pid:4405, start at 2025-04-04 20:22:10.773658263 +0800 CST m=+2.311307673 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:10.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:10.765 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:10.765 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432226576644829 TiDB Cluster ID. tikv_gc_leader_uuid 6580301910c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-tkdj8, pid:4468, start at 2025-04-04 20:22:10.770362679 +0800 CST m=+2.231823345 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:10.779 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:10.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:10.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark not exists for 15-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:22:14.073 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2025/04/04 20:22:14.078 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark8000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:14.154 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff successfully wait process cdc.test exit for 1-th time... [Fri Apr 4 20:22:14 CST 2025] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.58405842.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004401e60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118666945724422 MinTableBarrierTs:457118668269551619 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce {"id":"b9ddcdca-5455-4869-9e79-eb06f6a2f7ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee898da b9ddcdca-5455-4869-9e79-eb06f6a2f7ce /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118666945724422,"min-table-barrier-ts":457118668269551619,"admin-job-type":0} /tidb/cdc/default/default/task/position/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004401e60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118666945724422 MinTableBarrierTs:457118668269551619 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce {"id":"b9ddcdca-5455-4869-9e79-eb06f6a2f7ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee898da b9ddcdca-5455-4869-9e79-eb06f6a2f7ce /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118666945724422,"min-table-barrier-ts":457118668269551619,"admin-job-type":0} /tidb/cdc/default/default/task/position/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004401e60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118666945724422 MinTableBarrierTs:457118668269551619 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118666945724422, checkpointTs: 457118666945724422, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce {"id":"b9ddcdca-5455-4869-9e79-eb06f6a2f7ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee898da b9ddcdca-5455-4869-9e79-eb06f6a2f7ce /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118666945724422+ grep -q 'etcd info' ,"min-table-barrier-ts":457118668269551619,"admin-job-type":0} /tidb/cdc/default/default/task/position/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table test.finish_mark not exists for 24-th check, retry later table batch_add_table.finish_mark not exists for 16-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:15 CST 2025] <<<<<< run test case capture_session_done_during_task success! >>>>>> [2025/04/04 20:22:15.870 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark9000] [2025/04/04 20:22:15.870 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark9000] "is-owner": true, run task successfully [2025/04/04 20:22:16.455 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=1] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 20:22:16.455 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=7] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 20:22:16.455 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=140] [addr=127.0.0.1:9500] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 20:22:16.465 +08:00] [INFO] [main.go:196] ["genLock done"] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_client.go:147] ["[tso] exit tso dispatcher check loop"] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_dispatcher.go:490] ["[tso] exit tso connection contexts updater"] [dc-location=global] [2025/04/04 20:22:16.465 +08:00] [INFO] [pd_service_discovery.go:551] ["[pd] exit member loop due to context canceled"] [2025/04/04 20:22:16.465 +08:00] [INFO] [resource_manager_client.go:296] ["[resource manager] exit resource token dispatcher"] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_client.go:157] ["[tso] closing tso client"] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_dispatcher.go:158] ["[tso] exit tso deadline watcher"] [dc-location=global] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_dispatcher.go:264] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_stream.go:359] ["tsoStream.recvLoop ended"] [stream=127.0.0.1:2379-1] [error="rpc error: code = Canceled desc = context canceled"] [errorVerbose="rpc error: code = Canceled desc = context canceled\ngithub.com/tikv/pd/client.(*tsoStream).recvLoop\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20241111073742-238d4d79ea31/tso_stream.go:427\nruntime.goexit\n\t/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.4.linux-amd64/src/runtime/asm_amd64.s:1700"] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_dispatcher.go:201] ["[tso] exit tso dispatcher"] [dc-location=global] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_client.go:162] ["[tso] close tso client"] [2025/04/04 20:22:16.465 +08:00] [INFO] [tso_client.go:164] ["[tso] tso client is closed"] [2025/04/04 20:22:16.465 +08:00] [INFO] [pd_service_discovery.go:644] ["[pd] close pd service discovery client"] [2025/04/04 20:22:16.754 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2025/04/04 20:22:16.759 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark8000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:16.761 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118668227346446 : [2025/04/04 20:22:14.753 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=150] [jobID=151] [query="CREATE TABLE IF NOT EXISTS `finishmark8000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118668227346446] [2025/04/04 20:22:17.073 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark8000] [2025/04/04 20:22:17.073 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118668227346446] [2025/04/04 20:22:17.077 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:17.080 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:17.084 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:17.089 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 25-th check, retry later table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later owner pid: 4032 owner id b9ddcdca-5455-4869-9e79-eb06f6a2f7ce [Fri Apr 4 20:22:17 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.41374139.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 17-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:22:17.870 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2025/04/04 20:22:17.876 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark9000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:17.953 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:17 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9300bc29-bc84-4c68-a047-5179eb0fa7b3 {"id":"9300bc29-bc84-4c68-a047-5179eb0fa7b3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456ad7 9300bc29-bc84-4c68-a047-5179eb0fa7b3 /tidb/cdc/default/default/upstream/7489432223020059214 {"id":7489432223020059214,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9300bc29-bc84-4c68-a047-5179eb0fa7b3 {"id":"9300bc29-bc84-4c68-a047-5179eb0fa7b3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456ad7 9300bc29-bc84-4c68-a047-5179eb0fa7b3 /tidb/cdc/default/default/upstream/7489432223020059214 {"id":7489432223020059214,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9300bc29-bc84-4c68-a047-5179eb0fa7b3 {"id":"9300bc29-bc84-4c68-a047-5179eb0fa7b3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456ad7 9300bc29-bc84-4c68-a047-5179eb0fa7b3 /tidb/cdc/default/default/upstream/7489432223020059214 {"id":7489432223020059214,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.5899.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= Create changefeed successfully! ID: 03a9b11e-97b4-4117-8fa7-273b287bc4f5 Info: {"upstream_id":7489432223020059214,"namespace":"default","id":"03a9b11e-97b4-4117-8fa7-273b287bc4f5","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:22:18.284953757+08:00","start_ts":457118669397032963,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118669397032963,"checkpoint_ts":457118669397032963,"checkpoint_time":"2025-04-04 20:22:18.215"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... INSERT - Takes(s): 10.0, Count: 17222, OPS: 1723.8, Avg(us): 5775, Min(us): 844, Max(us): 96202, 95th(us): 55000, 99th(us): 88000 found first match line: 457118669118636046 : [2025/04/04 20:22:17.679 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=152] [jobID=153] [query="CREATE TABLE IF NOT EXISTS `finishmark9000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118669118636046] [2025/04/04 20:22:18.476 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark9000] [2025/04/04 20:22:18.476 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118669118636046] [2025/04/04 20:22:18.554 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:18.557 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:18.562 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:18.568 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 26-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table batch_add_table.finish_mark exists check diff successfully table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... [2025/04/04 20:22:17.891 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 5"] table test.finish_mark not exists for 27-th check, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 exists [Fri Apr 4 20:22:20 CST 2025] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.59845986.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cdabd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118669842677765 MinTableBarrierTs:457118669842677766 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4138a575-8a23-4181-82cf-8e7ad52b6a68 {"id":"4138a575-8a23-4181-82cf-8e7ad52b6a68","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769338} /tidb/cdc/default/__cdc_meta__/capture/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce {"id":"b9ddcdca-5455-4869-9e79-eb06f6a2f7ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee898da b9ddcdca-5455-4869-9e79-eb06f6a2f7ce /tidb/cdc/default/__cdc_meta__/owner/22319600bee89938 4138a575-8a23-4181-82cf-8e7ad52b6a68 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118669842677765,"min-table-barrier-ts":457118669842677766,"admin-job-type":0} /tidb/cdc/default/default/task/position/4138a575-8a23-4181-82cf-8e7ad52b6a68/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cdabd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118669842677765 MinTableBarrierTs:457118669842677766 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4138a575-8a23-4181-82cf-8e7ad52b6a68 {"id":"4138a575-8a23-4181-82cf-8e7ad52b6a68","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769338} /tidb/cdc/default/__cdc_meta__/capture/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce {"id":"b9ddcdca-5455-4869-9e79-eb06f6a2f7ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee898da b9ddcdca-5455-4869-9e79-eb06f6a2f7ce /tidb/cdc/default/__cdc_meta__/owner/22319600bee89938 4138a575-8a23-4181-82cf-8e7ad52b6a68 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118669842677765,"min-table-barrier-ts":457118669842677766,"admin-job-type":0} /tidb/cdc/default/default/task/position/4138a575-8a23-4181-82cf-8e7ad52b6a68/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cdabd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118669842677765 MinTableBarrierTs:457118669842677766 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4138a575-8a23-4181-82cf-8e7ad52b6a68 {"id":"4138a575-8a23-4181-82cf-8e7ad52b6a68","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769338} /tidb/cdc/default/__cdc_meta__/capture/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce {"id":"b9ddcdca-5455-4869-9e79-eb06f6a2f7ce","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee898da b9ddcdca-5455-4869-9e79-eb06f6a2f7ce /tidb/cdc/default/__cdc_meta__/owner/22319600bee89938 4138a575-8a23-4181-82cf-8e7ad52b6a68 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118669842677765,"min-table-barrier-ts":457118669842677766,"admin-job-type":0} /tidb/cdc/default/default/task/position/4138a575-8a23-4181-82cf-8e7ad52b6a68/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9ddcdca-5455-4869-9e79-eb06f6a2f7ce/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "b9ddcdca-5455-4869-9e79-eb06f6a2f7ce" | grep id cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m The 1 times to try to start tidb cluster... table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [2025/04/04 20:22:21.164 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark10000] [2025/04/04 20:22:21.164 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark10000] start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } "cluster-id": "default" "id": "4138a575-8a23-4181-82cf-8e7ad52b6a68", "cluster-id": "default" run task successfully [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table test.finish_mark not exists for 28-th check, retry later table test.t2 not exists for 1-th check, retry later [2025/04/04 20:22:23.165 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2025/04/04 20:22:23.171 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark10000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:23.172 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later capture_id: 4138a575-8a23-4181-82cf-8e7ad52b6a68 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 34313 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'b9ddcdca-5455-4869-9e79-eb06f6a2f7ce' b9ddcdca-5455-4869-9e79-eb06f6a2f7ce run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432273482665720 TiDB Cluster ID. tikv_gc_leader_uuid 65803023fe8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:6578, start at 2025-04-04 20:22:21.993075361 +0800 CST m=+2.773670307 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:22.000 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:21.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:21.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432273482665720 TiDB Cluster ID. tikv_gc_leader_uuid 65803023fe8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:6578, start at 2025-04-04 20:22:21.993075361 +0800 CST m=+2.773670307 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:22.000 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:21.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:21.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432260301615689 TiDB Cluster ID. tikv_gc_leader_uuid 65803024310000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-t-qj7hs, pid:6639, start at 2025-04-04 20:22:22.173827163 +0800 CST m=+2.882522741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:22.181 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:22.148 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:22.148 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {UpstreamID:7489432223020059214 Namespace:default ID:03a9b11e-97b4-4117-8fa7-273b287bc4f5 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:22:18.284953757 +0800 CST StartTs:457118669397032963 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f88510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118669410140165} {CheckpointTs:457118670786134024 MinTableBarrierTs:457118670786134024 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/728205e5-d67a-4daf-aa7f-cbbdc2a94a99 {"id":"728205e5-d67a-4daf-aa7f-cbbdc2a94a99","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769341} /tidb/cdc/default/__cdc_meta__/capture/9300bc29-bc84-4c68-a047-5179eb0fa7b3 {"id":"9300bc29-bc84-4c68-a047-5179eb0fa7b3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456ad7 9300bc29-bc84-4c68-a047-5179eb0fa7b3 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456b4e 728205e5-d67a-4daf-aa7f-cbbdc2a94a99 /tidb/cdc/default/default/changefeed/info/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"upstream-id":7489432223020059214,"namespace":"default","changefeed-id":"03a9b11e-97b4-4117-8fa7-273b287bc4f5","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:22:18.284953757+08:00","start-ts":457118669397032963,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118669410140165} /tidb/cdc/default/default/changefeed/status/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":457118670786134024,"min-table-barrier-ts":457118670786134024,"admin-job-type":0} /tidb/cdc/default/default/task/position/728205e5-d67a-4daf-aa7f-cbbdc2a94a99/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9300bc29-bc84-4c68-a047-5179eb0fa7b3/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432223020059214 {"id":7489432223020059214,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {UpstreamID:7489432223020059214 Namespace:default ID:03a9b11e-97b4-4117-8fa7-273b287bc4f5 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:22:18.284953757 +0800 CST StartTs:457118669397032963 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f88510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118669410140165} {CheckpointTs:457118670786134024 MinTableBarrierTs:457118670786134024 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/728205e5-d67a-4daf-aa7f-cbbdc2a94a99 {"id":"728205e5-d67a-4daf-aa7f-cbbdc2a94a99","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769341} /tidb/cdc/default/__cdc_meta__/capture/9300bc29-bc84-4c68-a047-5179eb0fa7b3 {"id":"9300bc29-bc84-4c68-a047-5179eb0fa7b3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456ad7 9300bc29-bc84-4c68-a047-5179eb0fa7b3 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456b4e 728205e5-d67a-4daf-aa7f-cbbdc2a94a99 /tidb/cdc/default/default/changefeed/info/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"upstream-id":7489432223020059214,"namespace":"default","changefeed-id":"03a9b11e-97b4-4117-8fa7-273b287bc4f5","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:22:18.284953757+08:00","start-ts":457118669397032963,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118669410140165} /tidb/cdc/default/default/changefeed/status/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":457118670786134024,"min-table-barrier-ts":457118670786134024,"admin-job-type":0} /tidb/cdc/default/default/task/position/728205e5-d67a-4daf-aa7f-cbbdc2a94a99/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9300bc29-bc84-4c68-a047-5179eb0fa7b3/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432223020059214 {"id":7489432223020059214,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {UpstreamID:7489432223020059214 Namespace:default ID:03a9b11e-97b4-4117-8fa7-273b287bc4f5 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:22:18.284953757 +0800 CST StartTs:457118669397032963 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f88510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118669410140165} {CheckpointTs:457118670786134024 MinTableBarrierTs:457118670786134024 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/728205e5-d67a-4daf-aa7f-cbbdc2a94a99 {"id":"728205e5-d67a-4daf-aa7f-cbbdc2a94a99","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769341} /tidb/cdc/default/__cdc_meta__/capture/9300bc29-bc84-4c68-a047-5179eb0fa7b3 {"id":"9300bc29-bc84-4c68-a047-5179eb0fa7b3","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769335} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456ad7 9300bc29-bc84-4c68-a047-5179eb0fa7b3 /tidb/cdc/default/__cdc_meta__/owner/22319600c0456b4e 728205e5-d67a-4daf-aa7f-cbbdc2a94a99 /tidb/cdc/default/default/changefeed/info/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"upstream-id":7489432223020059214,"namespace":"default","changefeed-id":"03a9b11e-97b4-4117-8fa7-273b287bc4f5","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:22:18.284953757+08:00","start-ts":457118669397032963,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118669410140165} /tidb/cdc/default/default/changefeed/status/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":457118670786134024,"min-table-barrier-ts":457118670786134024,"admin-job-type":0} /tidb/cdc/default/default/task/position/728205e5-d67a-4daf-aa7f-cbbdc2a94a99/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9300bc29-bc84-4c68-a047-5179eb0fa7b3/03a9b11e-97b4-4117-8fa7-273b287bc4f5 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432223020059214 {"id":7489432223020059214,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.t2 not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... [2025/04/04 20:22:24.964 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark11000] [2025/04/04 20:22:24.964 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark11000] table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later table test.finish_mark not exists for 29-th check, retry later table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Fri Apr 4 20:22:25 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.43054307.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:22:25.668 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2025/04/04 20:22:25.675 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark10000] /tmp/tidb_cdc_test/bank/cdcdown.log [Fri Apr 4 20:22:25 CST 2025] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.80168018.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:22:25.753 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118670639071246 : [2025/04/04 20:22:23.679 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=154] [jobID=155] [query="CREATE TABLE IF NOT EXISTS `finishmark10000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118670639071246] [2025/04/04 20:22:26.158 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark10000] [2025/04/04 20:22:26.158 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118670639071246] [2025/04/04 20:22:26.164 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:26.168 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... [2025/04/04 20:22:26.256 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:26.262 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.t2 not exists for 3-th check, retry later [2025/04/04 20:22:26.965 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later Verifying downstream PD is started... table test.finish_mark not exists for 30-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t2 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:28 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53f02659-dad3-45c9-83f5-a3414a13882f {"id":"53f02659-dad3-45c9-83f5-a3414a13882f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c06d63d7 53f02659-dad3-45c9-83f5-a3414a13882f /tidb/cdc/default/default/upstream/7489432273482665720 {"id":7489432273482665720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53f02659-dad3-45c9-83f5-a3414a13882f {"id":"53f02659-dad3-45c9-83f5-a3414a13882f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c06d63d7 53f02659-dad3-45c9-83f5-a3414a13882f /tidb/cdc/default/default/upstream/7489432273482665720 {"id":7489432273482665720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53f02659-dad3-45c9-83f5-a3414a13882f {"id":"53f02659-dad3-45c9-83f5-a3414a13882f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769346} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c06d63d7 53f02659-dad3-45c9-83f5-a3414a13882f /tidb/cdc/default/default/upstream/7489432273482665720 {"id":7489432273482665720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.finish_mark not exists for 31-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:22:28.970 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2025/04/04 20:22:29.057 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark11000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:29.058 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118671713861646 : [2025/04/04 20:22:27.779 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=156] [jobID=157] [query="CREATE TABLE IF NOT EXISTS `finishmark11000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118671713861646] [2025/04/04 20:22:29.564 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark11000] [2025/04/04 20:22:29.564 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118671713861646] table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 5845 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later [2025/04/04 20:22:29.653 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:29.657 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:29.662 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:29.668 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check_changefeed_state http://127.0.0.1:2379 b8be9574-356e-4545-a882-81ee56e9ef37 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=b8be9574-356e-4545-a882-81ee56e9ef37 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b8be9574-356e-4545-a882-81ee56e9ef37 -s + info='{ "upstream_id": 7489431832264407061, "namespace": "default", "id": "b8be9574-356e-4545-a882-81ee56e9ef37", "state": "finished", "checkpoint_tso": 457118669221134341, "checkpoint_time": "2025-04-04 20:22:17.544", "error": null }' + echo '{ "upstream_id": 7489431832264407061, "namespace": "default", "id": "b8be9574-356e-4545-a882-81ee56e9ef37", "state": "finished", "checkpoint_tso": 457118669221134341, "checkpoint_time": "2025-04-04 20:22:17.544", "error": null }' { "upstream_id": 7489431832264407061, "namespace": "default", "id": "b8be9574-356e-4545-a882-81ee56e9ef37", "state": "finished", "checkpoint_tso": 457118669221134341, "checkpoint_time": "2025-04-04 20:22:17.544", "error": null } ++ echo '{' '"upstream_id":' 7489431832264407061, '"namespace":' '"default",' '"id":' '"b8be9574-356e-4545-a882-81ee56e9ef37",' '"state":' '"finished",' '"checkpoint_tso":' 457118669221134341, '"checkpoint_time":' '"2025-04-04' '20:22:17.544",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7489431832264407061, '"namespace":' '"default",' '"id":' '"b8be9574-356e-4545-a882-81ee56e9ef37",' '"state":' '"finished",' '"checkpoint_tso":' 457118669221134341, '"checkpoint_time":' '"2025-04-04' '20:22:17.544",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:25 CST 2025] <<<<<< run test case changefeed_finish success! >>>>>> table test.table1 not exists for 1-th check, retry later [2025/04/04 20:22:30.174 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark12000] [2025/04/04 20:22:30.175 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark12000] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 5-th check, retry later INSERT - Takes(s): 20.0, Count: 28403, OPS: 1420.8, Avg(us): 7016, Min(us): 844, Max(us): 96202, 95th(us): 79000, 99th(us): 88000 shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table test.finish_mark not exists for 32-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030a2510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118670628847621 MinTableBarrierTs:457118670628847622 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2f4a1560-241f-4fa5-b310-1df1bac724da {"id":"2f4a1560-241f-4fa5-b310-1df1bac724da","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769345} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8999e 2f4a1560-241f-4fa5-b310-1df1bac724da /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118670628847621,"min-table-barrier-ts":457118670628847622,"admin-job-type":0} /tidb/cdc/default/default/task/position/2f4a1560-241f-4fa5-b310-1df1bac724da/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030a2510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118670628847621 MinTableBarrierTs:457118670628847622 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2f4a1560-241f-4fa5-b310-1df1bac724da {"id":"2f4a1560-241f-4fa5-b310-1df1bac724da","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769345} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8999e 2f4a1560-241f-4fa5-b310-1df1bac724da /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118670628847621,"min-table-barrier-ts":457118670628847622,"admin-job-type":0} /tidb/cdc/default/default/task/position/2f4a1560-241f-4fa5-b310-1df1bac724da/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 + grep -q 'failed to get info:' {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030a2510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118670628847621 MinTableBarrierTs:457118670628847622 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2f4a1560-241f-4fa5-b310-1df1bac724da {"id":"2f4a1560-241f-4fa5-b310-1df1bac724da","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769345} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8999e 2f4a1560-241f-4fa5-b310-1df1bac724da /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118670628847621,"min-table-barrier-ts":457118670628847622,"admin-job-type":0} /tidb/cdc/default/default/task/position/2f4a1560-241f-4fa5-b310-1df1bac724da/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table test.table1 exists table test.table2 not exists for 1-th check, retry later [2025/04/04 20:22:32.175 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2025/04/04 20:22:32.180 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark12000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:32.182 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118672998367246 : [2025/04/04 20:22:32.758 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=158] [jobID=159] [query="CREATE TABLE IF NOT EXISTS `finishmark12000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118672998367246] [2025/04/04 20:22:32.770 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark12000] [2025/04/04 20:22:32.770 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118672998367246] [2025/04/04 20:22:32.773 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432118507295449 TiDB Cluster ID. tikv_gc_leader_uuid 658030024b00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9258, start at 2025-04-04 20:21:47.447314816 +0800 CST m=+2.293463141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... table test.finish_mark not exists for 33-th check, retry later table test.t2 not exists for 6-th check, retry later [2025/04/04 20:22:32.857 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:32.861 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:32.869 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432316385497054 TiDB Cluster ID. tikv_gc_leader_uuid 6580302dd78000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:9650, start at 2025-04-04 20:22:32.059195929 +0800 CST m=+2.796850942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:32.068 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:32.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:32.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432316385497054 TiDB Cluster ID. tikv_gc_leader_uuid 6580302dd78000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:9650, start at 2025-04-04 20:22:32.059195929 +0800 CST m=+2.796850942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:32.068 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:32.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:32.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432301369739317 TiDB Cluster ID. tikv_gc_leader_uuid 6580302dd64000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:9709, start at 2025-04-04 20:22:32.041413131 +0800 CST m=+2.698105510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:32.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:32.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:32.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432312893241842 TiDB Cluster ID. tikv_gc_leader_uuid 6580302df60000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:6469, start at 2025-04-04 20:22:32.167183152 +0800 CST m=+2.714991445 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:32.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:32.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:32.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432312893241842 TiDB Cluster ID. tikv_gc_leader_uuid 6580302df60000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:6469, start at 2025-04-04 20:22:32.167183152 +0800 CST m=+2.714991445 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:32.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:32.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:32.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432314181731044 TiDB Cluster ID. tikv_gc_leader_uuid 6580302de94000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:6533, start at 2025-04-04 20:22:32.148215311 +0800 CST m=+2.610219220 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:32.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:32.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:32.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.table2 exists check diff failed 1-th time, retry later table test.finish_mark not exists for 34-th check, retry later table test.t2 not exists for 7-th check, retry later [2025/04/04 20:22:34.462 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark13000] [2025/04/04 20:22:34.462 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark13000] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432311262428761 TiDB Cluster ID. tikv_gc_leader_uuid 6580302f0a40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:7066, start at 2025-04-04 20:22:33.283550812 +0800 CST m=+2.440837493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:33.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:33.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:33.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432311262428761 TiDB Cluster ID. tikv_gc_leader_uuid 6580302f0a40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:7066, start at 2025-04-04 20:22:33.283550812 +0800 CST m=+2.440837493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:33.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:33.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:33.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432318510161364 TiDB Cluster ID. tikv_gc_leader_uuid 6580302f208000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:7135, start at 2025-04-04 20:22:33.382759757 +0800 CST m=+2.481380659 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:33.392 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:33.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:33.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored owner pid: 4310 owner id 2f4a1560-241f-4fa5-b310-1df1bac724da [Fri Apr 4 20:22:34 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.44294431.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11104.out cli tso query --pd=http://127.0.0.1:2379 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later [Fri Apr 4 20:22:35 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.77737775.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully table test.t2 exists check diff successfully [2025/04/04 20:22:36.462 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2025/04/04 20:22:36.553 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark13000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:36.555 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8401.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 35-th check, retry later found first match line: 457118674020728846 : [2025/04/04 20:22:36.778 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=160] [jobID=161] [query="CREATE TABLE IF NOT EXISTS `finishmark13000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118674020728846] [2025/04/04 20:22:37.179 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark13000] [2025/04/04 20:22:37.179 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118674020728846] [2025/04/04 20:22:37.257 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:37.261 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:37.265 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:37.270 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + set +x + tso='457118674054021122 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118674054021122 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:22:37 CST 2025] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1114911151.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 2' ____________________________________ *************************** 1. row *************************** id: 1 val: 1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:35 CST 2025] <<<<<< run test case ddl_puller_lag success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031d62d0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118674298863625 MinTableBarrierTs:457118674298863626 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2f4a1560-241f-4fa5-b310-1df1bac724da {"id":"2f4a1560-241f-4fa5-b310-1df1bac724da","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769345} /tidb/cdc/default/__cdc_meta__/capture/a161a158-594c-4ddc-9867-ec8c2e1b52fe {"id":"a161a158-594c-4ddc-9867-ec8c2e1b52fe","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769355} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8999e 2f4a1560-241f-4fa5-b310-1df1bac724da /tidb/cdc/default/__cdc_meta__/owner/22319600bee89a02 a161a158-594c-4ddc-9867-ec8c2e1b52fe /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118674298863625,"min-table-barrier-ts":457118674298863626,"admin-job-type":0} /tidb/cdc/default/default/task/position/2f4a1560-241f-4fa5-b310-1df1bac724da/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a161a158-594c-4ddc-9867-ec8c2e1b52fe/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031d62d0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118674298863625 MinTableBarrierTs:457118674298863626 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2f4a1560-241f-4fa5-b310-1df1bac724da {"id":"2f4a1560-241f-4fa5-b310-1df1bac724da","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769345} /tidb/cdc/default/__cdc_meta__/capture/a161a158-594c-4ddc-9867-ec8c2e1b52fe {"id":"a161a158-594c-4ddc-9867-ec8c2e1b52fe","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769355} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8999e 2f4a1560-241f-4fa5-b310-1df1bac724da /tidb/cdc/default/__cdc_meta__/owner/22319600bee89a02 a161a158-594c-4ddc-9867-ec8c2e1b52fe /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118674298863625,"min-table-barrier-ts":457118674298863626,"admin-job-type":0} /tidb/cdc/default/default/task/position/2f4a1560-241f-4fa5-b310+ grep -q 'failed to get info:' -1df1bac724da/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a161a158-594c-4ddc-9867-ec8c2e1b52fe/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031d62d0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118674298863625 MinTableBarrierTs:457118674298863626 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2f4a1560-241f-4fa5-b310-1df1bac724da {"id":"2f4a1560-241f-4fa5-b310-1df1bac724da","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769345} /tidb/cdc/default/__cdc_meta__/capture/a161a158-594c-4ddc-9867-ec8c2e1b52fe {"id":"a161a158-594c-4ddc-9867-ec8c2e1b52fe","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769355} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8999e 2f4a1560-241f-4fa5-b310-1df1bac724da /tidb/cdc/default/__cdc_meta__/owner/22319600bee89a02 a161a158-594c-4ddc-9867-ec8c2e1b52fe /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118674298863625,"min-table-barrier-ts":457118674298863626,"admin-job-type":0} /tidb/cdc/default/default/task/position/2f4a1560-241f-4fa5-b310-1df1bac724da/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a161a158-594c-4ddc-9867-ec8c2e1b52fe/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "2f4a1560-241f-4fa5-b310-1df1bac724da" | grep id cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:38 CST 2025] <<<<<< run test case resolve_lock success! >>>>>> INSERT - Takes(s): 30.1, Count: 40969, OPS: 1363.2, Avg(us): 7307, Min(us): 844, Max(us): 96202, 95th(us): 79000, 99th(us): 88000 + set +x + tso='457118674478956545 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118674478956545 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:22:39 CST 2025] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.84388440.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:38 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02393f4a-a121-4b0d-9cb3-1c9c91511623 {"id":"02393f4a-a121-4b0d-9cb3-1c9c91511623","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769356} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09767d3 02393f4a-a121-4b0d-9cb3-1c9c91511623 /tidb/cdc/default/default/upstream/7489432312893241842 {"id":7489432312893241842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02393f4a-a121-4b0d-9cb3-1c9c91511623 {"id":"02393f4a-a121-4b0d-9cb3-1c9c91511623","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769356} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09767d3 02393f4a-a121-4b0d-9cb3-1c9c91511623 /tidb/cdc/default/default/upstream/7489432312893241842 {"id":7489432312893241842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02393f4a-a121-4b0d-9cb3-1c9c91511623 {"id":"02393f4a-a121-4b0d-9cb3-1c9c91511623","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769356} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09767d3 02393f4a-a121-4b0d-9cb3-1c9c91511623 /tidb/cdc/default/default/upstream/7489432312893241842 {"id":7489432312893241842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:22:38 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.78277829.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 36-th check, retry later "cluster-id": "default" "id": "a161a158-594c-4ddc-9867-ec8c2e1b52fe", "cluster-id": "default" run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432334130116503 TiDB Cluster ID. tikv_gc_leader_uuid 65803032a140006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:24204, start at 2025-04-04 20:22:36.940550251 +0800 CST m=+2.073070185 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:36.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:36.982 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:36.982 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432334130116503 TiDB Cluster ID. tikv_gc_leader_uuid 65803032a140006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:24204, start at 2025-04-04 20:22:36.940550251 +0800 CST m=+2.073070185 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:36.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:36.982 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:36.982 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432338164640908 TiDB Cluster ID. tikv_gc_leader_uuid 65803032b940004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-wkbt2, pid:24275, start at 2025-04-04 20:22:37.032088618 +0800 CST m=+2.104052382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:37.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:37.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:37.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later Resume changefeed with checkpointTs3 457118674582765572 table test.table5 not exists for 1-th check, retry later [2025/04/04 20:22:39.457 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark14000] [2025/04/04 20:22:39.457 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark14000] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:40 GMT < Content-Length: 864 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23f75da2-ac57-4153-a50e-079b709f4a70 {"id":"23f75da2-ac57-4153-a50e-079b709f4a70","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769357} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0968453 23f75da2-ac57-4153-a50e-079b709f4a70 /tidb/cdc/default/default/upstream/7489432316385497054 {"id":7489432316385497054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23f75da2-ac57-4153-a50e-079b709f4a70 {"id":"23f75da2-ac57-4153-a50e-079b709f4a70","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769357} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0968453 23f75da2-ac57-4153-a50e-079b709f4a70 /tidb/cdc/default/default/upstream/7489432316385497054 {"id":7489432316385497054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/23f75da2-ac57-4153-a50e-079b709f4a70 {"id":"23f75da2-ac57-4153-a50e-079b709f4a70","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769357} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0968453 23f75da2-ac57-4153-a50e-079b709f4a70 /tidb/cdc/default/default/upstream/7489432316385497054 {"id":7489432316385497054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11207.out cli changefeed create --start-ts=457118674054021122 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. table test.finish_mark not exists for 37-th check, retry later Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489432316385497054,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:22:41.005447367+08:00","start_ts":457118674054021122,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118674054021122,"checkpoint_ts":457118674054021122,"checkpoint_time":"2025-04-04 20:22:35.980"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... capture_id: a161a158-594c-4ddc-9867-ec8c2e1b52fe + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.25610.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later table test.table5 not exists for 2-th check, retry later [2025/04/04 20:22:41.457 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2025/04/04 20:22:41.464 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark14000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:41.466 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:42 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e64abbc-b136-489b-a4c4-4dee060835c2 {"id":"4e64abbc-b136-489b-a4c4-4dee060835c2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769359} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09caaea 4e64abbc-b136-489b-a4c4-4dee060835c2 /tidb/cdc/default/default/upstream/7489432311262428761 {"id":7489432311262428761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e64abbc-b136-489b-a4c4-4dee060835c2 {"id":"4e64abbc-b136-489b-a4c4-4dee060835c2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769359} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09caaea 4e64abbc-b136-489b-a4c4-4dee060835c2 /tidb/cdc/default/default/upstream/7489432311262428761 {"id":7489432311262428761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e64abbc-b136-489b-a4c4-4dee060835c2 {"id":"4e64abbc-b136-489b-a4c4-4dee060835c2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769359} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09caaea 4e64abbc-b136-489b-a4c4-4dee060835c2 /tidb/cdc/default/default/upstream/7489432311262428761 {"id":7489432311262428761,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8503.out cli changefeed create --start-ts=457118674478956545 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:41 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02393f4a-a121-4b0d-9cb3-1c9c91511623 {"id":"02393f4a-a121-4b0d-9cb3-1c9c91511623","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769356} /tidb/cdc/default/__cdc_meta__/capture/d61b3f34-7ffd-463b-af11-20cbe1300e60 {"id":"d61b3f34-7ffd-463b-af11-20cbe1300e60","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769359} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09767d3 02393f4a-a121-4b0d-9cb3-1c9c91511623 /tidb/cdc/default/__cdc_meta__/owner/22319600c0976813 d61b3f34-7ffd-463b-af11-20cbe1300e60 /tidb/cdc/default/default/upstream/7489432312893241842 {"id":7489432312893241842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02393f4a-a121-4b0d-9cb3-1c9c91511623 {"id":"02393f4a-a121-4b0d-9cb3-1c9c91511623","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769356} /tidb/cdc/default/__cdc_meta__/capture/d61b3f34-7ffd-463b-af11-20cbe1300e60 {"id":"d61b3f34-7ffd-463b-af11-20cbe1300e60","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769359} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09767d3 02393f4a-a121-4b0d-9cb3-1c9c91511623 /tidb/cdc/default/__cdc_meta__/owner/22319600c0976813 d61b3f34-7ffd-463b-af11-20cbe1300e60 /tidb/cdc/default/default/upstream/7489432312893241842 {"id":7489432312893241842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02393f4a-a121-4b0d-9cb3-1c9c91511623 {"id":"02393f4a-a121-4b0d-9cb3-1c9c91511623","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769356} /tidb/cdc/default/__cdc_meta__/capture/d61b3f34-7ffd-463b-af11-20cbe1300e60 {"id":"d61b3f34-7ffd-463b-af11-20cbe1300e60","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769359} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c09767d3 02393f4a-a121-4b0d-9cb3-1c9c91511623 /tidb/cdc/default/__cdc_meta__/owner/22319600c0976813 d61b3f34-7ffd-463b-af11-20cbe1300e60 /tidb/cdc/default/default/upstream/7489432312893241842 + grep -q 'etcd info' {"id":7489432312893241842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later + set +x table test.simple not exists for 1-th check, retry later Create changefeed successfully! ID: 49cdcb6a-3891-4833-981a-4080ba0918db Info: {"upstream_id":7489432311262428761,"namespace":"default","id":"49cdcb6a-3891-4833-981a-4080ba0918db","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:22:42.60715816+08:00","start_ts":457118674478956545,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118674478956545,"checkpoint_ts":457118674478956545,"checkpoint_time":"2025-04-04 20:22:37.601"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... found first match line: 457118675357663246 : [2025/04/04 20:22:41.853 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=162] [jobID=163] [query="CREATE TABLE IF NOT EXISTS `finishmark14000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118675357663246] [2025/04/04 20:22:42.463 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark14000] [2025/04/04 20:22:42.463 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118675357663246] [2025/04/04 20:22:42.467 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:42.470 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:42.475 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:42.480 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.finish_mark not exists for 38-th check, retry later + set +x + tso='457118675509706753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118675509706753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:22:42 CST 2025] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2564325645.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later table test.table5 not exists for 3-th check, retry later + set +x go: downloading go1.23.4 (linux/amd64) [2025/04/04 20:22:43.860 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark15000] [2025/04/04 20:22:43.860 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark15000] check diff successfully table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432358118898116 TiDB Cluster ID. tikv_gc_leader_uuid 65803037c48000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:6667, start at 2025-04-04 20:22:42.206947793 +0800 CST m=+2.105850610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:42.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:42.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:42.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432358118898116 TiDB Cluster ID. tikv_gc_leader_uuid 65803037c48000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:6667, start at 2025-04-04 20:22:42.206947793 +0800 CST m=+2.105850610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:42.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:42.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:42.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432353484331030 TiDB Cluster ID. tikv_gc_leader_uuid 65803037e90000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:6734, start at 2025-04-04 20:22:42.360877705 +0800 CST m=+2.202504354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:42.397 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:42.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:42.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 39-th check, retry later table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:45 CST 2025] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> [2025/04/04 20:22:45.860 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2025/04/04 20:22:45.866 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark15000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:45.868 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later table test.table5 exists table test.table6 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:46 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82ead38e-771d-46d6-a606-c4c2f5a36ce1 {"id":"82ead38e-771d-46d6-a606-c4c2f5a36ce1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769363} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0aa77e9 82ead38e-771d-46d6-a606-c4c2f5a36ce1 /tidb/cdc/default/default/upstream/7489432334130116503 {"id":7489432334130116503,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82ead38e-771d-46d6-a606-c4c2f5a36ce1 {"id":"82ead38e-771d-46d6-a606-c4c2f5a36ce1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769363} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0aa77e9 82ead38e-771d-46d6-a606-c4c2f5a36ce1 /tidb/cdc/default/default/upstream/7489432334130116503 {"id":7489432334130116503,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82ead38e-771d-46d6-a606-c4c2f5a36ce1 {"id":"82ead38e-771d-46d6-a606-c4c2f5a36ce1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769363} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0aa77e9 82ead38e-771d-46d6-a606-c4c2f5a36ce1 /tidb/cdc/default/default/upstream/7489432334130116503 {"id":7489432334130116503,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.25701.out cli changefeed create --start-ts=457118675509706753 --sink-uri=mysql://root@127.0.0.1:3306/ =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Fri Apr 4 20:22:46 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.80748076.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 found first match line: 457118676327596047 : [2025/04/04 20:22:44.779 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=164] [jobID=165] [query="CREATE TABLE IF NOT EXISTS `finishmark15000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118676327596047] [2025/04/04 20:22:46.475 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark15000] [2025/04/04 20:22:46.475 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118676327596047] [2025/04/04 20:22:46.478 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:46.481 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:46.484 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:46.557 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Create changefeed successfully! ID: 37082b63-2be4-42a2-95f2-8812188cba9f Info: {"upstream_id":7489432334130116503,"namespace":"default","id":"37082b63-2be4-42a2-95f2-8812188cba9f","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T20:22:46.469492246+08:00","start_ts":457118675509706753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118675509706753,"checkpoint_ts":457118675509706753,"checkpoint_time":"2025-04-04 20:22:41.533"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Run finished, takes 37.369411157s INSERT - Takes(s): 37.4, Count: 50000, OPS: 1338.3, Avg(us): 7445, Min(us): 844, Max(us): 96202, 95th(us): 80000, 99th(us): 88000 table consistent_replicate_nfs.usertable exists wait process cdc.test exit for 1-th time... table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118674446974977, "checkpoint_time": "2025-04-04 20:22:37.479", "error": null }' + echo '{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118674446974977, "checkpoint_time": "2025-04-04 20:22:37.479", "error": null }' { "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118674446974977, "checkpoint_time": "2025-04-04 20:22:37.479", "error": null } ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118674446974977, '"checkpoint_time":' '"2025-04-04' '20:22:37.479",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118674446974977, '"checkpoint_time":' '"2025-04-04' '20:22:37.479",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432369428844827 TiDB Cluster ID. tikv_gc_leader_uuid 6580303a890000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g, pid:4059, start at 2025-04-04 20:22:45.048262754 +0800 CST m=+2.172533969 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:45.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:45.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:45.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432369428844827 TiDB Cluster ID. tikv_gc_leader_uuid 6580303a890000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g, pid:4059, start at 2025-04-04 20:22:45.048262754 +0800 CST m=+2.172533969 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:45.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:45.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:45.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432364959660155 TiDB Cluster ID. tikv_gc_leader_uuid 6580303a95c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-q-vx49g, pid:4136, start at 2025-04-04 20:22:45.128915986 +0800 CST m=+2.192203136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:45.135 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:45.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:45.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 table test.finish_mark not exists for 40-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later table test.table6 exists check diff successfully Resume changefeed with checkpointTs1 457118672210100233 + set +x Verifying downstream PD is started... table test.table3 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11490.out cli changefeed --changefeed-id custom-changefeed-name pause =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:22:48.868 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark16000] [2025/04/04 20:22:48.869 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark16000] Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.finish_mark not exists for 41-th check, retry later go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:49 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dfc093d-6f6d-4a4e-991d-50e47d7fd79f {"id":"2dfc093d-6f6d-4a4e-991d-50e47d7fd79f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769366} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0bf05d9 2dfc093d-6f6d-4a4e-991d-50e47d7fd79f /tidb/cdc/default/default/upstream/7489432358118898116 {"id":7489432358118898116,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dfc093d-6f6d-4a4e-991d-50e47d7fd79f {"id":"2dfc093d-6f6d-4a4e-991d-50e47d7fd79f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769366} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0bf05d9 2dfc093d-6f6d-4a4e-991d-50e47d7fd79f /tidb/cdc/default/default/upstream/7489432358118898116 {"id":7489432358118898116,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2dfc093d-6f6d-4a4e-991d-50e47d7fd79f {"id":"2dfc093d-6f6d-4a4e-991d-50e47d7fd79f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769366} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0bf05d9 2dfc093d-6f6d-4a4e-991d-50e47d7fd79f /tidb/cdc/default/default/upstream/7489432358118898116 {"id":7489432358118898116,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.8135.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/conf/changefeed.toml table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later [Fri Apr 4 20:22:49 CST 2025] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.54945496.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Create changefeed successfully! ID: 134d113c-5481-4fa0-9274-bafc1651987d Info: {"upstream_id":7489432358118898116,"namespace":"default","id":"134d113c-5481-4fa0-9274-bafc1651987d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:22:49.967864912+08:00","start_ts":457118677701754882,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!test.t_1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118677701754882,"checkpoint_ts":457118677701754882,"checkpoint_time":"2025-04-04 20:22:49.895"} PASS empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... table capture_suicide_while_balance_table.t4 exists table test.table3 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... [2025/04/04 20:22:50.869 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2025/04/04 20:22:50.874 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark16000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:50.876 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 42-th check, retry later + set +x table test.t_1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/tidwall/btree v1.7.0 go: downloading golang.org/x/sync v0.10.0 go: downloading google.golang.org/grpc v1.64.0 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/stretchr/testify v1.10.0 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/net v0.33.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/sys v0.28.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... found first match line: 457118677612101648 : [2025/04/04 20:22:49.853 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=166] [jobID=167] [query="CREATE TABLE IF NOT EXISTS `finishmark16000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118677612101648] [2025/04/04 20:22:51.576 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark16000] [2025/04/04 20:22:51.576 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118677612101648] [2025/04/04 20:22:51.656 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:51.660 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:51.666 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:51.671 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:22:52 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41db5c7a-0d7f-42cd-8680-f8533b0f993e {"id":"41db5c7a-0d7f-42cd-8680-f8533b0f993e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769369} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0cfc0d6 41db5c7a-0d7f-42cd-8680-f8533b0f993e /tidb/cdc/default/default/upstream/7489432369428844827 {"id":7489432369428844827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41db5c7a-0d7f-42cd-8680-f8533b0f993e {"id":"41db5c7a-0d7f-42cd-8680-f8533b0f993e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769369} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0cfc0d6 41db5c7a-0d7f-42cd-8680-f8533b0f993e /tidb/cdc/default/default/upstream/7489432369428844827 {"id":7489432369428844827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41db5c7a-0d7f-42cd-8680-f8533b0f993e {"id":"41db5c7a-0d7f-42cd-8680-f8533b0f993e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769369} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0cfc0d6 41db5c7a-0d7f-42cd-8680-f8533b0f993e /tidb/cdc/default/default/upstream/7489432369428844827 {"id":7489432369428844827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 12998050-c5bf-4140-b797-53494ffb5e67 Info: {"upstream_id":7489432369428844827,"namespace":"default","id":"12998050-c5bf-4140-b797-53494ffb5e67","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T20:22:52.52295113+08:00","start_ts":457118677540012033,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118677540012033,"checkpoint_ts":457118677540012033,"checkpoint_time":"2025-04-04 20:22:49.278"} go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 table test.table3 not exists for 3-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dolthub/maphash v0.1.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432118507295449 TiDB Cluster ID. tikv_gc_leader_uuid 658030024b00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9258, start at 2025-04-04 20:21:47.447314816 +0800 CST m=+2.293463141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.finish_mark not exists for 43-th check, retry later go: downloading github.com/coreos/go-systemd/v22 v22.5.0 % Total % Received % Xferd Average Speed Time Time Time Current go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading golang.org/x/term v0.27.0 Dload Upload Total Spent Left Speed go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/kylelemons/godebug v1.1.0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 100 140 100 35 100 105 8311 24934 --:--:-- --:--:-- --:--:-- 26250 { "status": true, "message": "" go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/mattn/go-runewidth v0.0.16 table force_replicate_table.t0 not exists for 1-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457118677029093381, "checkpoint_time": "2025-04-04 20:22:47.329", "error": null }' + echo '{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457118677029093381, "checkpoint_time": "2025-04-04 20:22:47.329", "error": null }' { "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457118677029093381, "checkpoint_time": "2025-04-04 20:22:47.329", "error": null } ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457118677029093381, '"checkpoint_time":' '"2025-04-04' '20:22:47.329",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457118677029093381, '"checkpoint_time":' '"2025-04-04' '20:22:47.329",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11555.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489432316385497054,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:22:41.005447367+08:00","start_ts":457118674054021122,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457118677029093381,"checkpoint_time":"2025-04-04 20:22:47.329"} PASS [2025/04/04 20:22:53.554 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark17000] [2025/04/04 20:22:53.554 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark17000] go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 coverage: 2.9% of statements in github.com/pingcap/tiflow/... table test.t_1 exists table test.t_2 exists table test.t_3 exists table test.t_4 exists table test.t_5 exists table test.t_6 exists table test.t_7 exists table test.t_8 exists table test.t_9 exists table test.t_10 exists table test.t_11 exists table test.t_12 exists table test.t_13 exists table test.t_14 exists table test.t_15 exists table test.t_16 exists table test.t_17 exists table test.t_18 exists table test.t_19 exists table test.t_20 exists table test.t_21 exists table test.t_22 exists table test.t_23 exists table test.t_24 exists table test.t_25 exists table test.t_26 exists table test.t_27 exists table test.t_28 exists table test.t_29 exists table test.t_30 exists table test.t_31 exists table test.t_32 exists table test.t_33 exists table test.t_34 exists table test.t_35 exists table test.t_36 exists table test.t_37 exists table test.t_38 exists table test.t_39 exists table test.t_40 exists table test.t_41 exists table test.t_42 exists table test.t_43 exists table test.t_44 exists table test.t_45 exists table test.t_46 exists table test.t_47 exists table test.t_48 exists table test.t_49 exists table test.t_50 exists table test.t_51 exists table test.t_52 exists table test.t_53 exists table test.t_54 exists table test.t_55 exists table test.t_56 exists table test.t_57 exists table test.t_58 exists table test.t_59 exists table test.t_60 exists table test.t_61 exists table test.t_62 exists table test.t_63 exists table test.t_64 exists table test.t_65 exists table test.t_66 exists table test.t_67 exists table test.t_68 exists table test.t_69 exists table test.t_70 exists table test.t_71 exists table test.t_72 exists table test.t_73 exists table test.t_74 exists table test.t_75 exists table test.t_76 exists table test.t_77 exists table test.t_78 exists table test.t_79 exists table test.t_80 exists table test.t_81 exists table test.t_82 exists table test.t_83 exists table test.t_84 exists table test.t_85 exists table test.t_86 exists table test.t_87 exists table test.t_88 exists table test.t_89 exists table test.t_90 exists table test.t_91 exists table test.t_92 exists table test.t_93 exists table test.t_94 exists table test.t_95 exists table test.t_96 exists table test.t_97 exists table test.t_98 exists table test.t_99 exists table test.t_100 exists table test.table3 exists table test.table4 not exists for 1-th check, retry later go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 44-th check, retry later table test.t_100 not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11599.out cli changefeed --changefeed-id custom-changefeed-name resume ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } }lease 22319600c0976813 revoked check diff successfully table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later [2025/04/04 20:22:55.554 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2025/04/04 20:22:55.564 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark17000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:22:55.565 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) go: downloading github.com/pierrec/lz4 v2.6.1+incompatible [Pipeline] // cache wait process cdc.test exit for 1-th time... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.table4 exists empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... [Pipeline] // node [Pipeline] } found first match line: 457118679046553615 : [2025/04/04 20:22:55.879 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=168] [jobID=169] [query="CREATE TABLE IF NOT EXISTS `finishmark17000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118679046553615] [2025/04/04 20:22:56.566 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark17000] [2025/04/04 20:22:56.566 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118679046553615] [2025/04/04 20:22:56.569 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:22:56.573 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:22:56.578 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] check diff failed 1-th time, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2025/04/04 20:22:56.664 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [Pipeline] // stage [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:22:56 CST 2025] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> table test.t_100 not exists for 2-th check, retry later + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432419196392627 TiDB Cluster ID. tikv_gc_leader_uuid 65803044bb40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:6720, start at 2025-04-04 20:22:55.507454634 +0800 CST m=+2.335987770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:55.515 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:55.519 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:55.519 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432419196392627 TiDB Cluster ID. tikv_gc_leader_uuid 65803044bb40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:6720, start at 2025-04-04 20:22:55.507454634 +0800 CST m=+2.335987770 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:55.515 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:55.519 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:55.519 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432418503197178 TiDB Cluster ID. tikv_gc_leader_uuid 65803044c90000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:6803, start at 2025-04-04 20:22:55.541828288 +0800 CST m=+2.307099981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:24:55.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:22:55.524 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:12:55.524 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.finish_mark not exists for 45-th check, retry later table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/jmespath/go-jmespath v0.4.0 [2025/04/04 20:22:58.365 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark18000] [2025/04/04 20:22:58.365 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark18000] nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.t_100 not exists for 3-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 46-th check, retry later table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "457118675825066005" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2025/04/04 20:22:58.069 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:22:58.071 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:22:58.071 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:22:58.071 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:22:58.071 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118675667779600] [resolvedTs=457118675825066005] [2025/04/04 20:22:58.071 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=414.162µs] [2025/04/04 20:22:58.073 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:22:58.073 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:22:58.073 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:22:58.078 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:22:58.081 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:22:58.082 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:22:58.083 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:22:58.083 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:22:58.083 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:22:58.083 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:22:58.083 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_df6c80b6-9f03-4c7a-b656-bb0393afc387_row_457118676349354017_49a4fce6-40c7-4d38-b0e8-b05a1b2d5989.log] [endTs=457118675825066005] [2025/04/04 20:22:58.084 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_df6c80b6-9f03-4c7a-b656-bb0393afc387_row_457118675853377554_d665541c-aaac-45d5-b8ce-c9550e833b59.log] [endTs=457118675825066005] [2025/04/04 20:22:58.085 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=13.652734ms] [2025/04/04 20:22:58.087 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:22:58.089 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:22:58.089 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/04 20:22:58.294 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}"] [checkpointTs=457118675825066005] [2025/04/04 20:22:58.295 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}"] [checkpointTs=457118675825066005] [2025/04/04 20:22:58.395 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=652] [appliedDDLCount=0] [currentCheckpoint=457118675825066005] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:22:58.395 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] Apply redo log successfully check diff successfully [Fri Apr 4 20:22:58 CST 2025] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 46229 [Fri Apr 4 20:22:59 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.81388140.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:00 CST 2025] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.87028704.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118680175083528, "checkpoint_time": "2025-04-04 20:22:59.330", "error": null }' + echo '{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118680175083528, "checkpoint_time": "2025-04-04 20:22:59.330", "error": null }' { "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118680175083528, "checkpoint_time": "2025-04-04 20:22:59.330", "error": null } ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118680175083528, '"checkpoint_time":' '"2025-04-04' '20:22:59.330",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118680175083528, '"checkpoint_time":' '"2025-04-04' '20:22:59.330",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11673.out cli changefeed --changefeed-id custom-changefeed-name remove [2025/04/04 20:23:00.365 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2025/04/04 20:23:00.458 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark18000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:23:00.460 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 47-th check, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457118680437227529 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.t_100 not exists for 4-th check, retry later found first match line: 457118680338399246 : [2025/04/04 20:23:00.953 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=170] [jobID=171] [query="CREATE TABLE IF NOT EXISTS `finishmark18000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118680338399246] [2025/04/04 20:23:01.569 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark18000] [2025/04/04 20:23:01.569 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118680338399246] [2025/04/04 20:23:01.573 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:23:01.577 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later [2025/04/04 20:23:01.658 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:23:01.664 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432433865419319 TiDB Cluster ID. tikv_gc_leader_uuid 658030497a0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:23421, start at 2025-04-04 20:23:00.349916798 +0800 CST m=+2.482040607 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:00.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:00.360 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:00.360 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432433865419319 TiDB Cluster ID. tikv_gc_leader_uuid 658030497a0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:23421, start at 2025-04-04 20:23:00.349916798 +0800 CST m=+2.482040607 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:00.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:00.360 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:00.360 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432427611896670 TiDB Cluster ID. tikv_gc_leader_uuid 658030499b0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:23485, start at 2025-04-04 20:23:00.484101618 +0800 CST m=+2.547884746 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:00.494 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:00.460 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:00.460 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass [2025/04/04 20:23:02.757 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark19000] [2025/04/04 20:23:02.757 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark19000] + set +x < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:02 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ba325cc-26a2-45eb-8050-252a7bf7e164 {"id":"1ba325cc-26a2-45eb-8050-252a7bf7e164","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e5d6 1ba325cc-26a2-45eb-8050-252a7bf7e164 /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ba325cc-26a2-45eb-8050-252a7bf7e164 {"id":"1ba325cc-26a2-45eb-8050-252a7bf7e164","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e5d6 1ba325cc-26a2-45eb-8050-252a7bf7e164 /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ba325cc-26a2-45eb-8050-252a7bf7e164 {"id":"1ba325cc-26a2-45eb-8050-252a7bf7e164","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e5d6 1ba325cc-26a2-45eb-8050-252a7bf7e164 /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8196.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7489432419196392627,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-04T20:23:03.03501263+08:00","start_ts":457118681129025538,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118681129025538,"checkpoint_ts":457118681129025538,"checkpoint_time":"2025-04-04 20:23:02.969"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later table test.finish_mark not exists for 48-th check, retry later wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.t_100 not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18620773-e602-4fea-b253-4ac64e4b34b7 {"id":"18620773-e602-4fea-b253-4ac64e4b34b7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600c06d65ba 18620773-e602-4fea-b253-4ac64e4b34b7 /tidb/cdc/default/default/changefeed/info/074753f7-65dc-4847-98da-c141bf2f6a79 {"upstream-id":7489432273482665720,"namespace":"default","changefeed-id":"074753f7-65dc-4847-98da-c141bf2f6a79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:22:28.86739252+08:00","start-ts":457118672210100233,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-04T20:23:00.59959897+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118680507219970} /tidb/cdc/default/default/changefeed/status/074753f7-65dc-4847-98da-c141bf2f6a79 {"checkpoint-ts":457118674753159189,"min-table-barrier-ts":457118674753159189,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489432273482665720 {"id":7489432273482665720,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18620773-e602-4fea-b253-4ac64e4b34b7 {"id":"18620773-e602-4fea-b253-4ac64e4b34b7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600c06d65ba 18620773-e602-4fea-b253-4ac64e4b34b7 /tidb/cdc/default/default/changefeed/info/074753f7-65dc-4847-98da-c141bf2f6a79 {"upstream-id":7489432273482665720,"namespace":"default","changefeed-id":"074753f7-65dc-4847-98da-c141bf2f6a79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:22:28.86739252+08:00","start-ts":457118672210100233,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-04T20:23:00.59959897+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118680507219970} /tidb/cdc/default/default/changefeed/status/074753f7-65dc-4847-98da-c141bf2f6a79 {"checkpoint-ts":457118674753159189,"min-table-barrier-ts":457118674753159189,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489432273482665720 {"id":7489432273482665720,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18620773-e602-4fea-b253-4ac64e4b34b7 {"id":"18620773-e602-4fea-b253-4ac64e4b34b7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600c06d65ba 18620773-e602-4fea-b253-4ac64e4b34b7 /tidb/cdc/default/default/changefeed/info/074753f7-65dc-4847-98da-c141bf2f6a79 {"upstream-id":7489432273482665720,"namespace":"default","changefeed-id":"074753f7-65dc-4847-98da-c141bf2f6a79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:22:28.86739252+08:00","start-ts":457118672210100233,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-04T20:23:00.59959897+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118680507219970} /tidb/cdc/default/default/changefeed/status/074753f7-65dc-4847-98da-c141bf2f6a79 {"checkpoint-ts":457118674753159189,"min-table-barrier-ts":457118674753159189,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489432273482665720 {"id":7489432273482665720,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 074753f7-65dc-4847-98da-c141bf2f6a79 failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=074753f7-65dc-4847-98da-c141bf2f6a79 + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 074753f7-65dc-4847-98da-c141bf2f6a79 -s + info='{ "upstream_id": 7489432273482665720, "namespace": "default", "id": "074753f7-65dc-4847-98da-c141bf2f6a79", "state": "failed", "checkpoint_tso": 457118674753159189, "checkpoint_time": "2025-04-04 20:22:38.647", "error": { "time": "2025-04-04T20:23:00.59959897+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7489432273482665720, "namespace": "default", "id": "074753f7-65dc-4847-98da-c141bf2f6a79", "state": "failed", "checkpoint_tso": 457118674753159189, "checkpoint_time": "2025-04-04 20:22:38.647", "error": { "time": "2025-04-04T20:23:00.59959897+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7489432273482665720, "namespace": "default", "id": "074753f7-65dc-4847-98da-c141bf2f6a79", "state": "failed", "checkpoint_tso": 457118674753159189, "checkpoint_time": "2025-04-04 20:22:38.647", "error": { "time": "2025-04-04T20:23:00.59959897+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489432273482665720, '"namespace":' '"default",' '"id":' '"074753f7-65dc-4847-98da-c141bf2f6a79",' '"state":' '"failed",' '"checkpoint_tso":' 457118674753159189, '"checkpoint_time":' '"2025-04-04' '20:22:38.647",' '"error":' '{' '"time":' '"2025-04-04T20:23:00.59959897+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489432273482665720, '"namespace":' '"default",' '"id":' '"074753f7-65dc-4847-98da-c141bf2f6a79",' '"state":' '"failed",' '"checkpoint_tso":' 457118674753159189, '"checkpoint_time":' '"2025-04-04' '20:22:38.647",' '"error":' '{' '"time":' '"2025-04-04T20:23:00.59959897+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully wait process cdc.test exit for 2-th time... check_changefeed_state http://127.0.0.1:2379 074753f7-65dc-4847-98da-c141bf2f6a79 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=074753f7-65dc-4847-98da-c141bf2f6a79 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 074753f7-65dc-4847-98da-c141bf2f6a79 -s + info='{ "upstream_id": 7489432273482665720, "namespace": "default", "id": "074753f7-65dc-4847-98da-c141bf2f6a79", "state": "normal", "checkpoint_tso": 457118681293651974, "checkpoint_time": "2025-04-04 20:23:03.597", "error": null }' + echo '{ "upstream_id": 7489432273482665720, "namespace": "default", "id": "074753f7-65dc-4847-98da-c141bf2f6a79", "state": "normal", "checkpoint_tso": 457118681293651974, "checkpoint_time": "2025-04-04 20:23:03.597", "error": null }' { "upstream_id": 7489432273482665720, "namespace": "default", "id": "074753f7-65dc-4847-98da-c141bf2f6a79", "state": "normal", "checkpoint_tso": 457118681293651974, "checkpoint_time": "2025-04-04 20:23:03.597", "error": null } ++ echo '{' '"upstream_id":' 7489432273482665720, '"namespace":' '"default",' '"id":' '"074753f7-65dc-4847-98da-c141bf2f6a79",' '"state":' '"normal",' '"checkpoint_tso":' 457118681293651974, '"checkpoint_time":' '"2025-04-04' '20:23:03.597",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432273482665720, '"namespace":' '"default",' '"id":' '"074753f7-65dc-4847-98da-c141bf2f6a79",' '"state":' '"normal",' '"checkpoint_tso":' 457118681293651974, '"checkpoint_time":' '"2025-04-04' '20:23:03.597",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case delete_owner_key [Fri Apr 4 20:23:04 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.48394841.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24805.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 49-th check, retry later [2025/04/04 20:23:04.757 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2025/04/04 20:23:04.768 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark19000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:23:04.770 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table test.t_100 not exists for 6-th check, retry later table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later found first match line: 457118681413189646 : [2025/04/04 20:23:04.978 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=172] [jobID=173] [query="CREATE TABLE IF NOT EXISTS `finishmark19000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118681413189646] [2025/04/04 20:23:05.766 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark19000] [2025/04/04 20:23:05.766 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118681413189646] [2025/04/04 20:23:05.778 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:23:05.781 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:23:05.786 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:23:05.858 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test1.table1 not exists for 1-th check, retry later table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11737.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name + set +x + tso='457118681559203841 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118681559203841 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_attributes.finish_mark not exists for 2-th check, retry later [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489432316385497054,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:23:06.204510828+08:00","start_ts":457118681957400578,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118681957400578,"checkpoint_ts":457118681957400578,"checkpoint_time":"2025-04-04 20:23:06.129"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 50-th check, retry later ***************** properties ***************** "workload"="core" "readproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="10000" "mysql.db"="move_table" "mysql.user"="root" "updateproportion"="0" "threadcount"="10" "dotransactions"="false" "insertproportion"="0" "operationcount"="0" "readallfields"="true" "scanproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" ********************************************** table test.t_100 not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:23:05.376 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.610358477s"] table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044b63f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118681914408963 MinTableBarrierTs:457118682176552965 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118682176552965, checkpointTs: 457118682176552965, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118682176552963, checkpointTs: 457118682176552963, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118682176552966, checkpointTs: 457118682176552965, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10f873ba-bb97-4767-a839-13aa73097a0a {"id":"10f873ba-bb97-4767-a839-13aa73097a0a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769384} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 10f873ba-bb97-4767-a839-13aa73097a0a /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118681914408965,"min-table-barrier-ts":457118682176552965,"admin-job-type":0} /tidb/cdc/default/default/task/position/10f873ba-bb97-4767-a839-13aa73097a0a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044b63f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118681914408963 MinTableBarrierTs:457118682176552965 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118682176552965, checkpointTs: 457118682176552965, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118682176552963, checkpointTs: 457118682176552963, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118682176552966, checkpointTs: 457118682176552965, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10f873ba-bb97-4767-a839-13aa73097a0a {"id":"10f873ba-bb97-4767-a839-13aa73097a0a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769384} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 10f873ba-bb97-4767-a839-13aa73097a0a /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118681914408965,"min-table-barrier-ts":457118682176552965,"admin-job-type":0} /tidb/cdc/default/default/task/position/10f873ba-bb97-4767-a839-13aa73097a0a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044b63f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118681914408963 MinTableBarrierTs:457118682176552965 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118682176552965, checkpointTs: 457118682176552965, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118682176552963, checkpointTs: 457118682176552963, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118682176552966, checkpointTs: 457118682176552965, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10f873ba-bb97-4767-a839-13aa73097a0a {"id":"10f873ba-bb97-4767-a839-13aa73097a0a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769384} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 10f873ba-bb97-4767-a839-13aa73097a0a /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118681914408965,"min-table-barrier-ts":457118682176552965,"admin-job-type":0} /tidb/cdc/default/default/task/position/10f873ba-bb97-4767-a839-13aa73097a0a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + set +x table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later [2025/04/04 20:23:07.457 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20001] [2025/04/04 20:23:07.457 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/04 20:23:07.457 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20002] [2025/04/04 20:23:07.458 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/04 20:23:07.458 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20003] [2025/04/04 20:23:07.458 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/04 20:23:07.462 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20004] [2025/04/04 20:23:07.462 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] table test1.table1 not exists for 2-th check, retry later table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later [2025/04/04 20:23:07.963 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark20000] [2025/04/04 20:23:07.963 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20004] [2025/04/04 20:23:07.963 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark20000] run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 table ddl_attributes.finish_mark not exists for 3-th check, retry later run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 "is-owner": true, run task successfully table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... table test.finish_mark not exists for 51-th check, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 2.242437256s INSERT - Takes(s): 2.2, Count: 10000, OPS: 4474.1, Avg(us): 2203, Min(us): 854, Max(us): 46043, 95th(us): 3000, 99th(us): 37000 [Fri Apr 4 20:23:09 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2489924901.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 table test.t_100 not exists for 8-th check, retry later table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... owner pid: 4844 owner id 10f873ba-bb97-4767-a839-13aa73097a0a default owner key /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 [Fri Apr 4 20:23:10 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.49724974.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test1.table1 exists table test1.table2 exists check diff successfully [2025/04/04 20:23:09.965 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2025/04/04 20:23:09.970 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark20000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/04 20:23:09.971 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457118682467794953 : [2025/04/04 20:23:08.928 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=174] [jobID=175] [query="CREATE TABLE IF NOT EXISTS `finishmark20000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457118682467794953] [2025/04/04 20:23:10.123 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark20000] [2025/04/04 20:23:10.124 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457118682467794953] [2025/04/04 20:23:10.126 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:23:10.129 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:23:10.133 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/04 20:23:10.136 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/04 20:23:10.137 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/04 20:23:10.137 +08:00] [INFO] [case.go:620] ["bank test finished"] [valid=20] [tries=20] [ratio=1] + cleanup_process cdc.test run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 table ddl_attributes.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118683058405385, "checkpoint_time": "2025-04-04 20:23:10.329", "error": null }' + echo '{ "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118683058405385, "checkpoint_time": "2025-04-04 20:23:10.329", "error": null }' { "upstream_id": 7489432316385497054, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118683058405385, "checkpoint_time": "2025-04-04 20:23:10.329", "error": null } ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118683058405385, '"checkpoint_time":' '"2025-04-04' '20:23:10.329",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432316385497054, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118683058405385, '"checkpoint_time":' '"2025-04-04' '20:23:10.329",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 52-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.t_100 not exists for 9-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432119985441098 TiDB Cluster ID. tikv_gc_leader_uuid 65803002190000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9179, start at 2025-04-04 20:21:47.269635329 +0800 CST m=+2.175174146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432118507295449 TiDB Cluster ID. tikv_gc_leader_uuid 658030024b00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-b2xfh, pid:9258, start at 2025-04-04 20:21:47.447314816 +0800 CST m=+2.293463141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:23:47.457 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:21:47.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:11:47.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14542.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit ++ date + echo '[Fri Apr 4 20:23:11 CST 2025] <<<<<< run test case bank success! >>>>>>' [Fri Apr 4 20:23:11 CST 2025] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster wait process cdc.test exit for 2-th time... run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11839.out cli unsafe delete-service-gc-safepoint start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:12 CST 2025] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS table ddl_attributes.finish_mark not exists for 5-th check, retry later coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:12 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24957.out cli changefeed create --start-ts=457118681559203841 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 5471bf6c-9bfe-4955-ab8c-9c5bb6a294af Info: {"upstream_id":7489432433865419319,"namespace":"default","id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:23:12.884329624+08:00","start_ts":457118681559203841,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118681559203841,"checkpoint_ts":457118681559203841,"checkpoint_time":"2025-04-04 20:23:04.610"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 53-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t_100 not exists for 10-th check, retry later table ddl_only_block_related_table.finish_mark exists + set +x + tso='457118683467087875 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118683467087875 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118683467087875 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri Apr 4 20:23:13 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1457714579.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044bc000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118683487010823 MinTableBarrierTs:457118683749154825 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10f873ba-bb97-4767-a839-13aa73097a0a {"id":"10f873ba-bb97-4767-a839-13aa73097a0a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769384} /tidb/cdc/default/__cdc_meta__/capture/9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e {"id":"9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 10f873ba-bb97-4767-a839-13aa73097a0a /tidb/cdc/default/__cdc_meta__/owner/22319600bee89ba4 9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118683487010823,"min-table-barrier-ts":457118683749154825,"admin-job-type":0} /tidb/cdc/default/default/task/position/10f873ba-bb97-4767-a839-13aa73097a0a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044bc000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118683487010823 MinTableBarrierTs:457118683749154825 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10f873ba-bb97-4767-a839-13aa73097a0a {"id":"10f873ba-bb97-4767-a839-13aa73097a0a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769384} /tidb/cdc/default/__cdc_meta__/capture/9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e {"id":"9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 10f873ba-bb97-4767-a839-13aa73097a0a /tidb/cdc/default/__cdc_meta__/owner/22319600bee89ba4 9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118683487010823,"min-table-barrier-ts":457118683749154825,"admin-job-type":0} /tidb/cdc/default/default/task/position/10f873ba-bb97-4767-a839-13aa73097a0a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044bc000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118683487010823 MinTableBarrierTs:457118683749154825 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10f873ba-bb97-4767-a839-13aa73097a0a {"id":"10f873ba-bb97-4767-a839-13aa73097a0a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769384} /tidb/cdc/default/__cdc_meta__/capture/9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e {"id":"9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89b46 10f873ba-bb97-4767-a839-13aa73097a0a /tidb/cdc/default/__cdc_meta__/owner/22319600bee89ba4 9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118683487010823,"min-table-barrier-ts":457118683749154825,"admin-job-type":0} /tidb/cdc/default/default/task/position/10f873ba-bb97-4767-a839-13aa73097a0a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "10f873ba-bb97-4767-a839-13aa73097a0a default" | grep id wait process 8143 exit for 1-th time... check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11863.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later wait process 8143 exit for 2-th time... reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_attributes.finish_mark not exists for 6-th check, retry later + set +x [Fri Apr 4 20:23:14 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2499424996.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... run task successfully /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8143) - No such process wait process 8143 exit for 3-th time... process 8143 already exit [Fri Apr 4 20:23:14 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.82768278.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "id": "9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e", run task successfully Verifying downstream PD is started... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table test.t_100 not exists for 11-th check, retry later wait process cdc.test exit for 2-th time... + set +x table test.finish_mark not exists for 54-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:16 CST 2025] <<<<<< run test case force_replicate_table success! >>>>>> capture_id: 9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e' 9fb24cd9-e3e4-49a3-87a5-7cbe8a3f570e run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 17060 0 --:--:-- --:--:-- --:--:-- 18200 "is_owner": false run task successfully table ddl_attributes.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489432119985441098 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:21:56.706440274 +0800 CST StartTs:457118662480101377 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038d86c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118663751499783} {CheckpointTs:457118666766155777 MinTableBarrierTs:457118666766155777 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78d6f51c-c6a9-4350-93fa-00a88cb63434 {"id":"78d6f51c-c6a9-4350-93fa-00a88cb63434","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769313} /tidb/cdc/default/__cdc_meta__/capture/b880f51f-da90-4795-9e1b-579b3b1b7477 {"id":"b880f51f-da90-4795-9e1b-579b3b1b7477","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfea11e7 78d6f51c-c6a9-4350-93fa-00a88cb63434 /tidb/cdc/default/__cdc_meta__/owner/22319600c12f17d2 b880f51f-da90-4795-9e1b-579b3b1b7477 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489432119985441098,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:21:56.706440274+08:00","start-ts":457118662480101377,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118663751499783} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457118666766155777,"min-table-barrier-ts":457118666766155777,"admin-job-type":0} /tidb/cdc/default/default/task/position/78d6f51c-c6a9-4350-93fa-00a88cb63434/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b880f51f-da90-4795-9e1b-579b3b1b7477/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432119985441098 {"id":7489432119985441098,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489432119985441098 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:21:56.706440274 +0800 CST StartTs:457118662480101377 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038d86c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118663751499783} {CheckpointTs:457118666766155777 MinTableBarrierTs:457118666766155777 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78d6f51c-c6a9-4350-93fa-00a88cb63434 {"id":"78d6f51c-c6a9-4350-93fa-00a88cb63434","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769313} /tidb/cdc/default/__cdc_meta__/capture/b880f51f-da90-4795-9e1b-579b3b1b7477 {"id":"b880f51f-da90-4795-9e1b-579b3b1b7477","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfea11e7 78d6f51c-c6a9-4350-93fa-00a88cb63434 /tidb/cdc/default/__cdc_meta__/owner/22319600c12f17d2 b880f51f-da90-4795-9e1b-579b3b1b7477 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489432119985441098,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:21:56.706440274+08:00","start-ts":457118662480101377,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118663751499783} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457118666766155777,"min-table-barrier-ts":457118666766155777,"admin-job-type":0} /tidb/cdc/default/default/task/position/78d6f51c-c6a9-4350-93fa-00a88cb63434/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b880f51f-da90-4795-9e1b-579b3b1b7477/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432119985441098 {"id":7489432119985441098,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489432119985441098 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:21:56.706440274 +0800 CST StartTs:457118662480101377 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038d86c0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118663751499783} {CheckpointTs:457118666766155777 MinTableBarrierTs:457118666766155777 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78d6f51c-c6a9-4350-93fa-00a88cb63434 {"id":"78d6f51c-c6a9-4350-93fa-00a88cb63434","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769313} /tidb/cdc/default/__cdc_meta__/capture/b880f51f-da90-4795-9e1b-579b3b1b7477 {"id":"b880f51f-da90-4795-9e1b-579b3b1b7477","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769393} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600bfea11e7 78d6f51c-c6a9-4350-93fa-00a88cb63434 /tidb/cdc/default/__cdc_meta__/owner/22319600c12f17d2 b880f51f-da90-4795-9e1b-579b3b1b7477 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489432119985441098,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:21:56.706440274+08:00","start-ts":457118662480101377,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118663751499783} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457118666766155777,"min-table-barrier-ts":457118666766155777,"admin-job-type":0} /tidb/cdc/default/default/task/position/78d6f51c-c6a9-4350-93fa-00a88cb63434/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b880f51f-da90-4795-9e1b-579b3b1b7477/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432119985441098 {"id":7489432119985441098,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118683467087875 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14642.out cli changefeed create --start-ts=457118683467087875 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 table test.t_100 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 55-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {UpstreamID:7489432433865419319 Namespace:default ID:5471bf6c-9bfe-4955-ab8c-9c5bb6a294af SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:12.884329624 +0800 CST StartTs:457118681559203841 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a233b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118683721891846} {CheckpointTs:457118682083491841 MinTableBarrierTs:457118684783575054 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ae9b302-243a-495e-973f-1b3f806bd736 {"id":"9ae9b302-243a-495e-973f-1b3f806bd736","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b58 9ae9b302-243a-495e-973f-1b3f806bd736 /tidb/cdc/default/default/changefeed/info/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"upstream-id":7489432433865419319,"namespace":"default","changefeed-id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:12.884329624+08:00","start-ts":457118681559203841,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118683721891846} /tidb/cdc/default/default/changefeed/status/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":457118682083491841,"min-table-barrier-ts":457118684783575054,"admin-job-type":0} /tidb/cdc/default/default/task/position/9ae9b302-243a-495e-973f-1b3f806bd736/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d65388e2-83a2-433b-aa64-9f7681136d23/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {UpstreamID:7489432433865419319 Namespace:default ID:5471bf6c-9bfe-4955-ab8c-9c5bb6a294af SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:12.884329624 +0800 CST StartTs:457118681559203841 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a233b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118683721891846} {CheckpointTs:457118682083491841 MinTableBarrierTs:457118684783575054 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ae9b302-243a-495e-973f-1b3f806bd736 {"id":"9ae9b302-243a-495e-973f-1b3f806bd736","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b58 9ae9b302-243a-495e-973f-1b3f806bd736 /tidb/cdc/default/default/changefeed/info/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"upstream-id":7489432433865419319,"namespace":"default","changefeed-id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:12.884329624+08:00","start-ts":457118681559203841,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118683721891846} /tidb/cdc/default/default/changefeed/status/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":457118682083491841,"min-table-barrier-ts":457118684783575054,"admin-job-type":0} /tidb/cdc/default/default/task/position/9ae9b302-243a-495e-973f-1b3f806bd736/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d65388e2-83a2-433b-aa64-9f7681136d23/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af + grep -q 'failed to get info:' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {UpstreamID:7489432433865419319 Namespace:default ID:5471bf6c-9bfe-4955-ab8c-9c5bb6a294af SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:12.884329624 +0800 CST StartTs:457118681559203841 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a233b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118683721891846} {CheckpointTs:457118682083491841 MinTableBarrierTs:457118684783575054 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ae9b302-243a-495e-973f-1b3f806bd736 {"id":"9ae9b302-243a-495e-973f-1b3f806bd736","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b58 9ae9b302-243a-495e-973f-1b3f806bd736 /tidb/cdc/default/default/changefeed/info/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"upstream-id":7489432433865419319,"namespace":"default","changefeed-id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:12.884329624+08:00","start-ts":457118681559203841,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118683721891846} /tidb/cdc/default/default/changefeed/status/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":457118682083491841,"min-table-barrier-ts":457118684783575054,"admin-job-type":0} /tidb/cdc/default/default/task/position/9ae9b302-243a-495e-973f-1b3f806bd736/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d65388e2-83a2-433b-aa64-9f7681136d23/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 4 20:23:17 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2507125073.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489432419196392627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:03.03501263 +0800 CST StartTs:457118681129025538 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d4a120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118681141870594} {CheckpointTs:457118684877684737 MinTableBarrierTs:457118684877684737 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/034ed796-82f7-4837-937f-0be10f485d40 {"id":"034ed796-82f7-4837-937f-0be10f485d40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e6a9 034ed796-82f7-4837-937f-0be10f485d40 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489432419196392627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:23:03.03501263+08:00","start-ts":457118681129025538,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118681141870594} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457118684877684737,"min-table-barrier-ts":457118684877684737,"admin-job-type":0} /tidb/cdc/default/default/task/position/034ed796-82f7-4837-937f-0be10f485d40/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489432419196392627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:03.03501263 +0800 CST StartTs:457118681129025538 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d4a120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118681141870594} {CheckpointTs:457118684877684737 MinTableBarrierTs:457118684877684737 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/034ed796-82f7-4837-937f-0be10f485d40 {"id":"034ed796-82f7-4837-937f-0be10f485d40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e6a9 034ed796-82f7-4837-937f-0be10f485d40 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489432419196392627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:23:03.03501263+08:00","start-ts":457118681129025538,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118681141870594} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457118684877684737,"min-table-barrier-ts":457118684877684737,"admin-job-type":0} /tidb/cdc/default/default/task/position/034ed796-82f7-4837-937f-0be10f485d40/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432419196392627 + grep -q 'failed to get info:' {"id":7489432419196392627,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489432419196392627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:03.03501263 +0800 CST StartTs:457118681129025538 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d4a120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118681141870594} {CheckpointTs:457118684877684737 MinTableBarrierTs:457118684877684737 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118684877684737, checkpointTs: 457118684877684737, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/034ed796-82f7-4837-937f-0be10f485d40 {"id":"034ed796-82f7-4837-937f-0be10f485d40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e6a9 034ed796-82f7-4837-937f-0be10f485d40 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489432419196392627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:23:03.03501263+08:00","start-ts":457118681129025538,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118681141870594} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table + grep -q 'etcd info' {"checkpoint-ts":457118684877684737,"min-table-barrier-ts":457118684877684737,"admin-job-type":0} /tidb/cdc/default/default/task/position/034ed796-82f7-4837-937f-0be10f485d40/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_attributes.finish_mark exists Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11951.out cli unsafe resolve-lock --region=28 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432507652717089 TiDB Cluster ID. tikv_gc_leader_uuid 6580305a1a4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:9236, start at 2025-04-04 20:23:17.37694134 +0800 CST m=+2.520877177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:17.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:17.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:17.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432507652717089 TiDB Cluster ID. tikv_gc_leader_uuid 6580305a1a4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:9236, start at 2025-04-04 20:23:17.37694134 +0800 CST m=+2.520877177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:17.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:17.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:17.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432510651114360 TiDB Cluster ID. tikv_gc_leader_uuid 6580305a198000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:9300, start at 2025-04-04 20:23:17.365867412 +0800 CST m=+2.427590485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:17.375 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:17.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:17.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 56-th check, retry later table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later Create changefeed successfully! ID: test-2 Info: {"upstream_id":7489432119985441098,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:23:19.289821391+08:00","start_ts":457118683467087875,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118683467087875,"checkpoint_ts":457118683467087875,"checkpoint_time":"2025-04-04 20:23:11.888"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:20 CST 2025] <<<<<< run test case ddl_attributes success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check_ts_not_forward ddl-only-block-related-table + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11984.out cli unsafe resolve-lock --region=28 --ts=457118684487352322 + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {UpstreamID:7489432433865419319 Namespace:default ID:5471bf6c-9bfe-4955-ab8c-9c5bb6a294af SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:12.884329624 +0800 CST StartTs:457118681559203841 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0016fae10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118683721891846} {CheckpointTs:457118682376568862 MinTableBarrierTs:457118685600940038 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ae9b302-243a-495e-973f-1b3f806bd736 {"id":"9ae9b302-243a-495e-973f-1b3f806bd736","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/capture/b9674dd6-d9d0-4f0c-b073-745ef81fd58a {"id":"b9674dd6-d9d0-4f0c-b073-745ef81fd58a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769398} /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b58 9ae9b302-243a-495e-973f-1b3f806bd736 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b9f b9674dd6-d9d0-4f0c-b073-745ef81fd58a /tidb/cdc/default/default/changefeed/info/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"upstream-id":7489432433865419319,"namespace":"default","changefeed-id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:12.884329624+08:00","start-ts":457118681559203841,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118683721891846} /tidb/cdc/default/default/changefeed/status/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":457118682376568862,"min-table-barrier-ts":457118685600940038,"admin-job-type":0} /tidb/cdc/default/default/task/position/9ae9b302-243a-495e-973f-1b3f806bd736/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9674dd6-d9d0-4f0c-b073-745ef81fd58a/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d65388e2-83a2-433b-aa64-9f7681136d23/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {UpstreamID:7489432433865419319 Namespace:default ID:5471bf6c-9bfe-4955-ab8c-9c5bb6a294af SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:12.884329624 +0800 CST StartTs:457118681559203841 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0016fae10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118683721891846} {CheckpointTs:457118682376568862 MinTableBarrierTs:457118685600940038 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ae9b302-243a-495e-973f-1b3f806bd736 {"id":"9ae9b302-243a-495e-973f-1b3f806bd736","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/capture/b9674dd6-d9d0-4f0c-b073-745ef81fd58a {"id":"b9674dd6-d9d0-4f0c-b073-745ef81fd58a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769398} /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b58 9ae9b302-243a-495e-973f-1b3f806bd736 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b9f b9674dd6-d9d0-4f0c-b073-745ef81fd58a /tidb/cdc/default/default/changefeed/info/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"upstream-id":7489432433865419319,"namespace":"default","changefeed-id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:12.884329624+08:00","start-ts":457118681559203841,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118683721891846} /tidb/cdc/default/default/changefeed/status/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":457118682376568862,"min-table-barrier-ts":457118685600940038,"admin-job-type":0} /tidb/cdc/default/default/task/position/9ae9b302-243a-495e-973f-1b3f806bd736/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9674dd6-d9d0-4f0c-b073-745ef81fd58a/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d65388e2-83a2-433b-aa64-9f7681136d23/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {UpstreamID:7489432433865419319 Namespace:default ID:5471bf6c-9bfe-4955-ab8c-9c5bb6a294af SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:12.884329624 +0800 CST StartTs:457118681559203841 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0016fae10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118683721891846} {CheckpointTs:457118682376568862 MinTableBarrierTs:457118685600940038 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ae9b302-243a-495e-973f-1b3f806bd736 {"id":"9ae9b302-243a-495e-973f-1b3f806bd736","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769394} /tidb/cdc/default/__cdc_meta__/capture/b9674dd6-d9d0-4f0c-b073-745ef81fd58a {"id":"b9674dd6-d9d0-4f0c-b073-745ef81fd58a","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769398} /tidb/cdc/default/__cdc_meta__/capture/d65388e2-83a2-433b-aa64-9f7681136d23 {"id":"d65388e2-83a2-433b-aa64-9f7681136d23","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b05 d65388e2-83a2-433b-aa64-9f7681136d23 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b58 9ae9b302-243a-495e-973f-1b3f806bd736 /tidb/cdc/default/__cdc_meta__/owner/22319600c1046b9f b9674dd6-d9d0-4f0c-b073-745ef81fd58a /tidb/cdc/default/default/changefeed/info/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"upstream-id":7489432433865419319,"namespace":"default","changefeed-id":"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:12.884329624+08:00","start-ts":457118681559203841,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118683721891846} /tidb/cdc/default/default/changefeed/status/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":457118682376568862,"min-table-barrier-ts":457118685600940038,"admin-job-type":0} /tidb/cdc/default/default/task/position/9ae9b302-243a-495e-973f-1b3f806bd736/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9674dd6-d9d0-4f0c-b073-745ef81fd58a/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/d65388e2-83a2-433b-aa64-9f7681136d23/5471bf6c-9bfe-4955-ab8c-9c5bb6a294af {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432433865419319 {"id":7489432433865419319,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table test.finish_mark not exists for 57-th check, retry later table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/go-mysql-org/go-mysql v1.9.1 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.10569.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/jinzhu/now v1.1.5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432515505163827 TiDB Cluster ID. tikv_gc_leader_uuid 6580305cd380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:9919, start at 2025-04-04 20:23:20.175641121 +0800 CST m=+2.283537633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:20.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:20.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:20.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432515505163827 TiDB Cluster ID. tikv_gc_leader_uuid 6580305cd380012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:9919, start at 2025-04-04 20:23:20.175641121 +0800 CST m=+2.283537633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:20.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:20.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:20.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432522224662272 TiDB Cluster ID. tikv_gc_leader_uuid 6580305cee40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:10000, start at 2025-04-04 20:23:20.262539203 +0800 CST m=+2.296575179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:20.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:20.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:20.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 151k 4402 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/go-playground/locales v0.14.1 [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv table test.finish_mark exists [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x + tso='457118686157996033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118686157996033 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:23:23 CST 2025] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.1060510607.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... [Fri Apr 4 20:23:24 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1126211264.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 3-th time... } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 163k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v9.0.0-beta.1-9-g4c631d595", "git_hash": "4c631d5951854ac91765b729fe3849d8e8d5ed8a", "id": "a4fc7cf3-976d-4a15-857e-eddb0fbade63", "pid": 11154, "is_owner": true }wait process cdc.test exit for 1-th time... table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:23:25 CST 2025] <<<<<< run test case tidb_mysql_test success! >>>>>> [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:26 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0 {"id":"af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769403} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c14ab3f1 af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0 /tidb/cdc/default/default/upstream/7489432507652717089 {"id":7489432507652717089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0 {"id":"af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769403} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c14ab3f1 af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0 /tidb/cdc/default/default/upstream/7489432507652717089 {"id":7489432507652717089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0 {"id":"af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769403} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c14ab3f1 af3a6ca1-6410-4b51-8a78-cc40a2a3aaf0 /tidb/cdc/default/default/upstream/7489432507652717089 {"id":7489432507652717089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.10664.out cli changefeed create --start-ts=457118686157996033 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml table batch_add_table.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:26 CST 2025] <<<<<< run test case cli_with_auth success! >>>>>> go: downloading github.com/ardielle/ardielle-go v1.5.2 Create changefeed successfully! ID: 0886d1cf-c901-4fb7-a841-d73b404757fb Info: {"upstream_id":7489432507652717089,"namespace":"default","id":"0886d1cf-c901-4fb7-a841-d73b404757fb","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:23:27.114302095+08:00","start_ts":457118686157996033,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118686157996033,"checkpoint_ts":457118686157996033,"checkpoint_time":"2025-04-04 20:23:22.153"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac0f96bc-dff4-442f-843b-82117a642881 {"id":"ac0f96bc-dff4-442f-843b-82117a642881","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769404} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1546cd7 ac0f96bc-dff4-442f-843b-82117a642881 /tidb/cdc/default/default/upstream/7489432515505163827 {"id":7489432515505163827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac0f96bc-dff4-442f-843b-82117a642881 {"id":"ac0f96bc-dff4-442f-843b-82117a642881","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769404} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1546cd7 ac0f96bc-dff4-442f-843b-82117a642881 /tidb/cdc/default/default/upstream/7489432515505163827 {"id":7489432515505163827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac0f96bc-dff4-442f-843b-82117a642881 {"id":"ac0f96bc-dff4-442f-843b-82117a642881","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769404} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1546cd7 ac0f96bc-dff4-442f-843b-82117a642881 /tidb/cdc/default/default/upstream/7489432515505163827 {"id":7489432515505163827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later ***************** properties ***************** "threadcount"="10" "dotransactions"="false" "operationcount"="0" "recordcount"="5000" "fieldcount"="100" "updateproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "workload"="core" "scanproportion"="0" "readproportion"="0" "mysql.user"="root" "mysql.db"="consistent_replicate_storage_file" "readallfields"="true" "requestdistribution"="uniform" "mysql.port"="4000" ********************************************** table batch_add_table.finish_mark not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x ***************** properties ***************** "scanproportion"="0" "mysql.user"="root" "recordcount"="10" "readallfields"="true" "mysql.db"="testSync" "workload"="core" "dotransactions"="false" "updateproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "mysql.host"="127.0.0.1" "mysql.port"="4000" "insertproportion"="0" "operationcount"="0" "readproportion"="0" ********************************************** Run finished, takes 10.689731ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1864.6, Avg(us): 2058, Min(us): 1137, Max(us): 5342, 95th(us): 6000, 99th(us): 6000 [2025/04/04 20:23:26.503 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:46' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.507 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:45' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.511 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:45' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.511 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:46' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.515 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:47' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.517 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:48' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.518 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:47' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.524 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:55' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.530 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:57' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.534 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:33:56' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.614 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:34:28' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.621 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:34:18' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.622 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:34:19' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.625 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:34:29' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.702 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:26.716 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '20:34:50' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:27.011 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:27.039 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:27.101 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:27.417 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/04 20:23:27.846 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] table processor_delay.t50 exists Run finished, takes 1.834439109s INSERT - Takes(s): 1.8, Count: 5000, OPS: 2738.0, Avg(us): 3355, Min(us): 1230, Max(us): 55411, 95th(us): 3000, 99th(us): 48000 table consistent_replicate_storage_file.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } table batch_add_table.finish_mark not exists for 3-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 4-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432571592914816 TiDB Cluster ID. tikv_gc_leader_uuid 65803068664000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:6843, start at 2025-04-04 20:23:32.003098674 +0800 CST m=+2.263771600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:32.010 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:31.993 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:31.993 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432571592914816 TiDB Cluster ID. tikv_gc_leader_uuid 65803068664000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:6843, start at 2025-04-04 20:23:32.003098674 +0800 CST m=+2.263771600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:32.010 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:31.993 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:31.993 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432568237220732 TiDB Cluster ID. tikv_gc_leader_uuid 658030685c4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:6910, start at 2025-04-04 20:23:31.970177173 +0800 CST m=+2.176014962 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:31.978 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:31.985 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:31.985 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:34 CST 2025] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> [2025/04/04 20:23:33.841 +08:00] [WARN] [diff.go:557] ["checksum is not equal"] [table=`test`.`itest`] [where="((TRUE) AND TRUE)"] [sourceChecksum=3439541884] [targetChecksum=2248548504] [getSourceChecksumCost=6.270802ms] [getTargetChecksumCost=3.467249ms] [2025/04/04 20:23:33.908 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 0, b: 0, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:26, }"] [2025/04/04 20:23:33.925 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:27, a: 1, b: 1, }"] [2025/04/04 20:23:34.025 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 2, b: 2, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:28, }"] [2025/04/04 20:23:34.125 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:29, a: 3, b: 3, }"] [2025/04/04 20:23:34.225 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:31, a: 5, b: 5, }"] [2025/04/04 20:23:34.325 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 6, b: 6, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:32, }"] [2025/04/04 20:23:34.426 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 8, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:21, a: 8, }"] [2025/04/04 20:23:34.526 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:23:35, a: 9, b: 9, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:34.626 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:23:37, a: 11, b: 11, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] table batch_add_table.finish_mark not exists for 5-th check, retry later [2025/04/04 20:23:34.726 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 12, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:38, a: 12, }"] [2025/04/04 20:23:34.826 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 14, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:40, a: 14, }"] [2025/04/04 20:23:34.926 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:23:41, a: 15, b: 15, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:35.027 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 16, b: 16, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:42, }"] [2025/04/04 20:23:35.127 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 17, b: 17, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:43, }"] [2025/04/04 20:23:35.227 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:23:45, a: 19, b: 19, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:35.327 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:23:47, a: 21, b: 21, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:35.427 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:26:48, a: 22, b: 22, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:35.527 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: HQIdu9a, d: 20:23:49, a: 23, b: 23, }"] [2025/04/04 20:23:35.628 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 24, b: 24, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:50, }"] [2025/04/04 20:23:35.728 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 25, b: 25, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:51, }"] [2025/04/04 20:23:35.828 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 28, b: 28, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:54, }"] table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... [2025/04/04 20:23:35.928 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 29, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:55, a: 29, }"] [2025/04/04 20:23:36.028 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:56, a: 30, b: 30, }"] [2025/04/04 20:23:36.128 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 31, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:57, a: 31, }"] [Fri Apr 4 20:23:36 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.81468148.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/04 20:23:36.229 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 32, b: 32, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:23:58, }"] [2025/04/04 20:23:36.329 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:24:00, a: 34, b: 34, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:36.429 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 36, b: 36, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:02, }"] table batch_add_table.finish_mark not exists for 6-th check, retry later [2025/04/04 20:23:36.529 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 37, b: 37, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:03, }"] [2025/04/04 20:23:36.629 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 38, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:04, a: 38, }"] [2025/04/04 20:23:36.729 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:24:05, a: 39, b: 39, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:36.830 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:24:06, a: 40, b: 40, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:36.930 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 41, b: 41, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:07, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:23:37.030 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 42, b: 42, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:08, }"] [2025/04/04 20:23:37.130 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:24:09, a: 43, b: 43, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:37.230 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 46, b: 46, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:12, }"] [2025/04/04 20:23:37.331 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 47, b: 47, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:13, }"] [2025/04/04 20:23:37.431 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 48, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:14, a: 48, }"] [2025/04/04 20:23:37.531 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:24:16, a: 50, b: 50, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:37.631 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 51, b: 51, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:17, }"] [2025/04/04 20:23:37.731 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 52, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:18, a: 52, }"] [2025/04/04 20:23:37.832 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ d: 20:24:20, a: 54, b: 54, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: w6B3SVx, }"] [2025/04/04 20:23:37.932 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 55, b: 55, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:21, }"] table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/04 20:23:38.032 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 56, b: 56, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:22, }"] [2025/04/04 20:23:38.132 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:24:24, a: 58, b: 58, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:38.232 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:24:25, a: 59, b: 59, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:38.332 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 60, b: 60, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:26, }"] [2025/04/04 20:23:38.433 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 61, b: 61, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:27, }"] table batch_add_table.finish_mark not exists for 7-th check, retry later [2025/04/04 20:23:38.533 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:28, a: 62, b: 62, }"] [2025/04/04 20:23:38.633 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 63, b: 63, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:29, }"] [2025/04/04 20:23:38.733 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 64, b: 64, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:30, }"] [2025/04/04 20:23:38.833 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 66, b: 66, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:32, }"] [2025/04/04 20:23:38.934 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:24:35, a: 69, b: 69, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:39.034 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 70, b: 70, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:36, }"] [2025/04/04 20:23:39.134 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:24:37, a: 71, b: 71, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:39 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2704b3b1-678e-4930-bd82-8953b9ba8eae {"id":"2704b3b1-678e-4930-bd82-8953b9ba8eae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769416} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ead4 2704b3b1-678e-4930-bd82-8953b9ba8eae /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2704b3b1-678e-4930-bd82-8953b9ba8eae {"id":"2704b3b1-678e-4930-bd82-8953b9ba8eae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769416} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ead4 2704b3b1-678e-4930-bd82-8953b9ba8eae /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2704b3b1-678e-4930-bd82-8953b9ba8eae {"id":"2704b3b1-678e-4930-bd82-8953b9ba8eae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769416} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ead4 2704b3b1-678e-4930-bd82-8953b9ba8eae /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: beed2760-3bba-40f4-bf61-df055e02e635 Info: {"upstream_id":7489432571592914816,"namespace":"default","id":"beed2760-3bba-40f4-bf61-df055e02e635","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:23:39.330649881+08:00","start_ts":457118690624929798,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118690624929798,"checkpoint_ts":457118690624929798,"checkpoint_time":"2025-04-04 20:23:39.193"} table kill_owner_with_ddl.t1 not exists for 1-th check, retry later [2025/04/04 20:23:39.234 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:24:38, a: 72, b: 72, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:39.334 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 73, b: 73, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:39, }"] [2025/04/04 20:23:39.435 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 74, b: 74, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IlzI5fdb, d: 20:24:40, }"] [2025/04/04 20:23:39.535 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 75, b: 75, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:41, }"] [2025/04/04 20:23:39.635 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 76, b: 76, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:42, }"] nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/04 20:23:39.735 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 77, b: 77, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:43, }"] [2025/04/04 20:23:39.835 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 78, b: 78, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:44, }"] [2025/04/04 20:23:39.935 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 79, b: 79, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:45, }"] [2025/04/04 20:23:40.036 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:46, a: 80, b: 80, }"] [2025/04/04 20:23:40.136 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 81, b: 81, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:47, }"] start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later [2025/04/04 20:23:40.236 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:24:52, a: 86, b: 86, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:40.336 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:24:53, a: 87, b: 87, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:40.436 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 88, b: 88, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:54, }"] [2025/04/04 20:23:40.537 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 91, b: 91, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:57, }"] [2025/04/04 20:23:40.637 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 92, b: 92, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:58, }"] [2025/04/04 20:23:40.737 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 93, b: 93, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:24:59, }"] table batch_add_table.finish_mark not exists for 8-th check, retry later [2025/04/04 20:23:40.837 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 94, b: 94, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:00, }"] [2025/04/04 20:23:40.937 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:25:01, a: 95, b: 95, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:41.038 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:02, a: 96, b: 96, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:41.138 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 99, b: 99, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:05, }"] [2025/04/04 20:23:41.238 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 100, b: 100, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:06, }"] [2025/04/04 20:23:41.338 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 102, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:08, a: 102, }"] [2025/04/04 20:23:41.438 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:25:10, a: 104, b: 104, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:41.538 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 105, b: 105, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:11, }"] [2025/04/04 20:23:41.639 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 106, b: 106, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:12, }"] [2025/04/04 20:23:41.739 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 107, b: 107, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:13, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table kill_owner_with_ddl.t1 exists [2025/04/04 20:23:41.839 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 108, b: 108, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:14, }"] [2025/04/04 20:23:41.939 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 109, b: 109, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:15, }"] wait process 8151 exit for 1-th time... [2025/04/04 20:23:42.039 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:16, a: 110, b: 110, }"] [2025/04/04 20:23:42.140 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 111, b: 111, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:17, }"] [2025/04/04 20:23:42.240 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 112, b: 112, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:18, }"] table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass wait process cdc.test exit for 1-th time... [2025/04/04 20:23:42.340 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:25:19, a: 113, b: 113, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:42.440 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 114, b: 114, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:20, }"] [2025/04/04 20:23:42.540 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 116, b: 116, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:22, }"] [2025/04/04 20:23:42.641 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:34:49, a: 117, b: 117, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:42.741 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 118, b: 118, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:24, }"] wait process 8151 exit for 2-th time... table batch_add_table.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 2-th time... [2025/04/04 20:23:42.841 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 119, b: 119, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:32, }"] [2025/04/04 20:23:42.941 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 120, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:26, a: 120, }"] /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8151) - No such process wait process 8151 exit for 3-th time... process 8151 already exit check_capture_count http://127.0.0.1:2379 0 [2025/04/04 20:23:43.041 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:27, a: 121, b: 121, }"] [2025/04/04 20:23:43.141 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:25:28, a: 122, b: 122, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:43.241 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:29, a: 123, b: 123, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 20:23:43 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.82958297.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:43.342 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:30, a: 124, b: 124, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:43.442 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 125, b: 125, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:31, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_resign_owner [Fri Apr 4 20:23:43 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.54405442.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_resign_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_resign_owner.server1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... [2025/04/04 20:23:43.542 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:32, a: 126, b: 126, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:43.642 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 128, b: 128, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:34, }"] [2025/04/04 20:23:43.742 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 129, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:51, a: 129, }"] [2025/04/04 20:23:43.843 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 130, b: 130, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:04, }"] [2025/04/04 20:23:43.943 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:25:39, a: 133, b: 133, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later [2025/04/04 20:23:44.043 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 134, b: 134, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:20, }"] [2025/04/04 20:23:44.144 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:41, a: 135, b: 135, }"] [2025/04/04 20:23:44.244 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 137, b: 137, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:43, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:23:44.344 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:44, a: 138, b: 138, }"] [2025/04/04 20:23:44.444 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 140, b: 140, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:46, }"] [2025/04/04 20:23:44.544 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 141, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:47, a: 141, }"] table batch_add_table.finish_mark not exists for 10-th check, retry later [2025/04/04 20:23:44.644 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:48, a: 142, b: 142, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:44.745 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:25:49, a: 143, b: 143, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:44.845 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:50, a: 144, b: 144, }"] [2025/04/04 20:23:44.945 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 145, b: 145, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:51, }"] [2025/04/04 20:23:45.045 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 146, b: 146, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:52, }"] pass test: get status pass test: health [2025/04/04 20:23:38.304 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info=null] [2025/04/04 20:23:40.854 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info="{\"upstream_id\":7489431659457890943,\"namespace\":\"test\",\"id\":\"changefeed-test-v2-black-hole-1\",\"sink_uri\":\"blackhole:\",\"create_time\":\"2025-04-04T20:23:38.269382819+08:00\",\"start_ts\":457118690373271555,\"config\":{\"memory_quota\":1073741824,\"case_sensitive\":false,\"force_replicate\":false,\"ignore_ineligible_table\":false,\"check_gc_safe_point\":true,\"enable_sync_point\":false,\"bdr_mode\":false,\"sync_point_interval\":600000000000,\"sync_point_retention\":86400000000000,\"filter\":{\"rules\":[\"*.*\"]},\"mounter\":{\"worker_num\":16},\"sink\":{\"csv\":{\"delimiter\":\",\",\"quote\":\"\\\"\",\"null\":\"\\\\N\",\"include_commit_ts\":false},\"transaction_atomicity\":\"\",\"encoder_concurrency\":32,\"terminator\":\"\\r\\n\",\"date_separator\":\"day\",\"enable_partition_separator\":true,\"content_compatible\":false,\"send_bootstrap_interval_in_sec\":120,\"send_bootstrap_in_msg_count\":10000,\"send_bootstrap_to_all_partition\":true,\"debezium_disable_schema\":false,\"debezium\":{\"output_old_value\":true},\"open\":{\"output_old_value\":true}},\"consistent\":{\"level\":\"none\",\"max_log_size\":64,\"flush_interval\":2000,\"meta_flush_interval\":200,\"encoding_worker_num\":16,\"flush_worker_num\":8,\"storage\":\"\",\"use_file_backend\":false},\"scheduler\":{\"enable_table_across_nodes\":false,\"region_threshold\":100000,\"write_key_threshold\":0},\"integrity\":{\"integrity_check_level\":\"none\",\"corruption_handle_level\":\"warn\"}},\"state\":\"normal\",\"creator_version\":\"v9.0.0-beta.1-9-g4c631d595\",\"resolved_ts\":0,\"checkpoint_ts\":457118690975940612,\"checkpoint_time\":\"2025-04-04 20:23:40.532\"}"] [2025/04/04 20:23:45.145 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 147, b: 147, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:53, }"] [2025/04/04 20:23:45.246 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:54, a: 148, b: 148, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:45.346 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:55, a: 149, b: 149, }"] [2025/04/04 20:23:45.446 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 150, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:25:56, a: 150, }"] [2025/04/04 20:23:45.546 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:57, a: 151, b: 151, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:45.605 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info="{\"upstream_id\":7489431659457890943,\"namespace\":\"test\",\"id\":\"changefeed-test-v2-black-hole-1\",\"sink_uri\":\"blackhole:\",\"create_time\":\"2025-04-04T20:23:38.269382819+08:00\",\"start_ts\":457118690373271555,\"admin_job_type\":1,\"config\":{\"memory_quota\":1123450,\"case_sensitive\":false,\"force_replicate\":false,\"ignore_ineligible_table\":false,\"check_gc_safe_point\":false,\"enable_sync_point\":false,\"bdr_mode\":false,\"sync_point_interval\":600000000000,\"sync_point_retention\":86400000000000,\"filter\":{\"rules\":[\"a.d\",\"b.x\"],\"ignore_txn_start_ts\":[1,2,3],\"event_filters\":[{\"matcher\":[\"test.worker\"],\"ignore_event\":[\"update\"],\"ignore_sql\":[\"^drop\",\"add column\"],\"ignore_insert_value_expr\":\"id \\u003e= 100\",\"ignore_update_new_value_expr\":\"gender = 'male'\",\"ignore_update_old_value_expr\":\"age \\u003c 18\",\"ignore_delete_value_expr\":\"id \\u003e 100\"}]},\"mounter\":{\"worker_num\":17},\"sink\":{\"protocol\":\"arvo\",\"csv\":{\"delimiter\":\",\",\"quote\":\"\\\"\",\"null\":\"\\\\N\",\"include_commit_ts\":false},\"column_selectors\":[{\"matcher\":[\"a.b\"],\"columns\":[\"c\"]}],\"transaction_atomicity\":\"table\",\"encoder_concurrency\":32,\"terminator\":\"a\",\"date_separator\":\"day\",\"enable_partition_separator\":true,\"content_compatible\":true,\"send_bootstrap_interval_in_sec\":120,\"send_bootstrap_in_msg_count\":10000,\"send_bootstrap_to_all_partition\":true,\"debezium_disable_schema\":true,\"debezium\":{\"output_old_value\":true},\"open\":{\"output_old_value\":true}},\"consistent\":{\"level\":\"none\",\"max_log_size\":64,\"flush_interval\":2000,\"meta_flush_interval\":200,\"encoding_worker_num\":31,\"flush_worker_num\":18,\"storage\":\"\",\"use_file_backend\":false},\"scheduler\":{\"enable_table_across_nodes\":false,\"region_threshold\":13,\"write_key_threshold\":0},\"integrity\":{\"integrity_check_level\":\"none\",\"corruption_handle_level\":\"warn\"}},\"state\":\"stopped\",\"creator_version\":\"v9.0.0-beta.1-9-g4c631d595\",\"resolved_ts\":457118690975940612,\"checkpoint_ts\":457118690975940612,\"checkpoint_time\":\"2025-04-04 20:23:40.532\"}"] [2025/04/04 20:23:45.646 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:17, a: 152, b: 152, }"] [2025/04/04 20:23:45.746 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:25:59, a: 153, b: 153, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore [2025/04/04 20:23:45.847 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 154, b: 154, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:00, }"] [2025/04/04 20:23:45.947 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 155, b: 155, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:01, }"] [2025/04/04 20:23:46.047 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 156, b: 156, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:02, }"] table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:23:46.147 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:26:03, a: 157, b: 157, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:46.247 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 158, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:04, a: 158, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00373e120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692355080196 MinTableBarrierTs:457118692355080196 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692355080196, checkpointTs: 457118692355080196, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd565e05-67bc-462f-a173-6bc2543464f6 {"id":"dd565e05-67bc-462f-a173-6bc2543464f6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182eb55 dd565e05-67bc-462f-a173-6bc2543464f6 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692355080196,"min-table-barrier-ts":457118692355080196,"admin-job-type":0} /tidb/cdc/default/default/task/position/dd565e05-67bc-462f-a173-6bc2543464f6/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00373e120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692355080196 MinTableBarrierTs:457118692355080196 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692355080196, checkpointTs: 457118692355080196, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd565e05-67bc-462f-a173-6bc2543464f6 {"id":"dd565e05-67bc-462f-a173-6bc2543464f6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182eb55 dd565e05-67bc-462f-a173-6bc2543464f6 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692355080196,"min-table-barrier-ts":457118692355080196,"admin-job-type":0} /tidb/cdc/default/default/task/position/dd565e05-67bc-462f-a173-6bc2543464f6/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00373e120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692355080196 MinTableBarrierTs:457118692355080196 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692355080196, checkpointTs: 457118692355080196, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd565e05-67bc-462f-a173-6bc2543464f6 {"id":"dd565e05-67bc-462f-a173-6bc2543464f6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182eb55 dd565e05-67bc-462f-a173-6bc2543464f6 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692355080196,"min-table-barrier-ts":457118692355080196,"admin-job-type":0} /tidb/cdc/default/default/task/position/dd565e05-67bc-462f-a173-6bc2543464f6/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2025/04/04 20:23:46.347 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 159, b: 159, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:02, }"] [2025/04/04 20:23:46.448 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 160, b: 160, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:06, }"] [2025/04/04 20:23:46.548 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 161, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:33, a: 161, }"] table batch_add_table.finish_mark not exists for 11-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00428ec60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118692413276163 MinTableBarrierTs:457118692413276164 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118692413276164, checkpointTs: 457118692413276164, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692413276163, checkpointTs: 457118692413276163, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118692413276166, checkpointTs: 457118692413276164, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee {"id":"a226a456-03ea-4a17-bde9-f4d5c2e6a9ee","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89d45 a226a456-03ea-4a17-bde9-f4d5c2e6a9ee /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118692413276163,"min-table-barrier-ts":457118692413276164,"admin-job-type":0} /tidb/cdc/default/default/task/position/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00428ec60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118692413276163 MinTableBarrierTs:457118692413276164 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118692413276164, checkpointTs: 457118692413276164, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692413276163, checkpointTs: 457118692413276163, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118692413276166, checkpointTs: 457118692413276164, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee {"id":"a226a456-03ea-4a17-bde9-f4d5c2e6a9ee","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89d45 a226a456-03ea-4a17-bde9-f4d5c2e6a9ee /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118692413+ grep -q 'failed to get info:' 276163,"min-table-barrier-ts":457118692413276164,"admin-job-type":0} /tidb/cdc/default/default/task/position/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00428ec60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118692413276163 MinTableBarrierTs:457118692413276164 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118692413276164, checkpointTs: 457118692413276164, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692413276163, checkpointTs: 457118692413276163, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118692413276166, checkpointTs: 457118692413276164, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee {"id":"a226a456-03ea-4a17-bde9-f4d5c2e6a9ee","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89d45 a226a456-03ea-4a17-bde9-f4d5c2e6a9ee /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118692413276163,"min-table-barrier-ts":457118692413276164,"admin-job-type":0} /tidb/cdc/default/default/task/position/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee/330a1306-950b-43d9-93e3-10dbf8731a51 + grep -q 'etcd info' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2025/04/04 20:23:46.648 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 162, b: 162, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:08, }"] [2025/04/04 20:23:46.748 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 163, b: 163, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:09, }"] [2025/04/04 20:23:46.848 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:26:10, a: 164, b: 164, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:46.949 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 165, b: 165, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:11, }"] [2025/04/04 20:23:47.049 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:12, a: 166, b: 166, }"] [2025/04/04 20:23:47.149 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:26:13, a: 167, b: 167, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:47.249 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:14, a: 168, b: 168, }"] [2025/04/04 20:23:47.349 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 170, b: 170, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:16, }"] [2025/04/04 20:23:47.450 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 171, b: 171, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:17, }"] [2025/04/04 20:23:47.550 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 172, b: 172, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:18, }"] start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:23:47.650 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 173, b: 173, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:19, }"] [2025/04/04 20:23:47.750 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 175, b: 175, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:21, }"] table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later [2025/04/04 20:23:47.850 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 176, b: 176, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:22, }"] [2025/04/04 20:23:47.950 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:23, a: 177, b: 177, }"] [2025/04/04 20:23:48.051 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 178, b: 178, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:24, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully pass test: changefeed apis VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432630677090175 TiDB Cluster ID. tikv_gc_leader_uuid 65803076bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:7327, start at 2025-04-04 20:23:46.696133856 +0800 CST m=+2.773322735 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:46.705 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:46.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:46.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432630677090175 TiDB Cluster ID. tikv_gc_leader_uuid 65803076bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:7327, start at 2025-04-04 20:23:46.696133856 +0800 CST m=+2.773322735 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:46.705 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:46.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:46.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432634185836456 TiDB Cluster ID. tikv_gc_leader_uuid 65803076c940009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:7402, start at 2025-04-04 20:23:46.774730505 +0800 CST m=+2.773354376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:46.784 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:46.775 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:46.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/04 20:23:48.151 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 179, b: 179, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:25, }"] [2025/04/04 20:23:48.251 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:26:26, a: 180, b: 180, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:48.351 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:29, a: 183, b: 183, }"] [2025/04/04 20:23:48.203 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info=null] table batch_add_table.finish_mark not exists for 12-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:23:48.451 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:26:32, a: 186, b: 186, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:48.551 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 187, b: 187, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:33, }"] [2025/04/04 20:23:48.652 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 188, b: 188, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:34, }"] [2025/04/04 20:23:48.752 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:26:35, a: 189, b: 189, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:48.852 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 190, b: 190, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:36, }"] [2025/04/04 20:23:45.784 +08:00] [INFO] [main.go:61] ["table mover started"] [2025/04/04 20:23:45.786 +08:00] [INFO] [main.go:165] ["new cluster initialized"] [2025/04/04 20:23:45.787 +08:00] [DEBUG] [main.go:191] ["retrieved owner ID"] [ownerID=d65388e2-83a2-433b-aa64-9f7681136d23] [2025/04/04 20:23:45.787 +08:00] [DEBUG] [main.go:198] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2025/04/04 20:23:45.787 +08:00] [DEBUG] [main.go:209] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2025/04/04 20:23:45.914 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af] [captureID=9ae9b302-243a-495e-973f-1b3f806bd736] [processorDetail="{\"table_ids\":[]}"] [2025/04/04 20:23:46.064 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af] [captureID=b9674dd6-d9d0-4f0c-b073-745ef81fd58a] [processorDetail="{\"table_ids\":[]}"] [2025/04/04 20:23:46.214 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af] [captureID=d65388e2-83a2-433b-aa64-9f7681136d23] [processorDetail="{\"table_ids\":[114,116]}"] [2025/04/04 20:23:46.214 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"9ae9b302-243a-495e-973f-1b3f806bd736\":[],\"b9674dd6-d9d0-4f0c-b073-745ef81fd58a\":[],\"d65388e2-83a2-433b-aa64-9f7681136d23\":[{\"ID\":114,\"Changefeed\":\"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af\"},{\"ID\":116,\"Changefeed\":\"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af\"}]}"] [2025/04/04 20:23:46.214 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af&target-cp-id=b9674dd6-d9d0-4f0c-b073-745ef81fd58a&table-id=114"] [2025/04/04 20:23:46.264 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=114] [2025/04/04 20:23:46.264 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af&target-cp-id=b9674dd6-d9d0-4f0c-b073-745ef81fd58a&table-id=116"] [2025/04/04 20:23:46.314 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=116] [2025/04/04 20:23:46.314 +08:00] [INFO] [main.go:113] ["all tables are moved"] [sourceCapture=d65388e2-83a2-433b-aa64-9f7681136d23] [targetCapture=b9674dd6-d9d0-4f0c-b073-745ef81fd58a] table move_table.check1 exists check diff successfully Verifying downstream PD is started... [2025/04/04 20:23:48.952 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 191, b: 191, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:37, }"] [2025/04/04 20:23:49.052 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:26:38, a: 192, b: 192, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:49.153 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 193, b: 193, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:39, }"] [2025/04/04 20:23:49.253 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 194, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:40, a: 194, }"] [2025/04/04 20:23:49.353 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 195, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:41, a: 195, }"] [2025/04/04 20:23:49.453 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 196, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:42, a: 196, }"] [2025/04/04 20:23:49.553 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 197, b: 197, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:43, }"] [2025/04/04 20:23:49.653 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:26:44, a: 198, b: 198, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:49.754 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:19, a: 199, b: 199, }"] [2025/04/04 20:23:49.854 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:26:50, a: 200, b: 200, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] owner pid: 5445 owner id a226a456-03ea-4a17-bde9-f4d5c2e6a9ee [Fri Apr 4 20:23:49 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.55685570.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_resign_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_resign_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:49.954 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:51, a: 201, b: 201, }"] [2025/04/04 20:23:50.054 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 202, b: 202, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:52, }"] skip some DDLs in tidb v4.0.x [Fri Apr 4 20:23:50 CST 2025] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.87138715.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:50.154 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:56, a: 206, b: 206, }"] [2025/04/04 20:23:50.254 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 207, b: 207, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:57, }"] [2025/04/04 20:23:50.355 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 208, b: 208, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:58, }"] table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432636279251966 TiDB Cluster ID. tikv_gc_leader_uuid 65803078ad00011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:12906, start at 2025-04-04 20:23:48.692947909 +0800 CST m=+2.693329088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:48.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:48.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:48.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432636279251966 TiDB Cluster ID. tikv_gc_leader_uuid 65803078ad00011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:12906, start at 2025-04-04 20:23:48.692947909 +0800 CST m=+2.693329088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:48.729 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:48.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:48.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432640259941116 TiDB Cluster ID. tikv_gc_leader_uuid 65803078300000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:12983, start at 2025-04-04 20:23:48.174812617 +0800 CST m=+2.110023988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:48.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:48.160 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:48.160 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:23:50.455 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 209, b: 209, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:26:59, }"] [2025/04/04 20:23:50.555 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:00, a: 210, b: 210, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:50.655 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:01, a: 211, b: 211, }"] [2025/04/04 20:23:50.755 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:02, a: 212, b: 212, }"] [2025/04/04 20:23:50.856 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 213, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:03, a: 213, }"] pass test: delete changefeed apis pass test: capture apis table batch_add_table.finish_mark not exists for 13-th check, retry later [2025/04/04 20:23:50.956 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:27:04, a: 214, b: 214, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:51.056 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 215, b: 215, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:05, }"] pass test: processor apis [2025/04/04 20:23:51.156 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 216, b: 216, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:06, }"] [2025/04/04 20:23:51.256 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 218, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:08, a: 218, }"] [2025/04/04 20:23:51.357 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 219, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:09, a: 219, }"] [2025/04/04 20:23:51.370 +08:00] [INFO] [main.go:61] ["table mover started"] [2025/04/04 20:23:51.372 +08:00] [INFO] [main.go:165] ["new cluster initialized"] [2025/04/04 20:23:51.373 +08:00] [DEBUG] [main.go:191] ["retrieved owner ID"] [ownerID=d65388e2-83a2-433b-aa64-9f7681136d23] [2025/04/04 20:23:51.373 +08:00] [DEBUG] [main.go:198] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2025/04/04 20:23:51.373 +08:00] [DEBUG] [main.go:209] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2025/04/04 20:23:51.514 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af] [captureID=9ae9b302-243a-495e-973f-1b3f806bd736] [processorDetail="{\"table_ids\":[]}"] [2025/04/04 20:23:51.457 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 220, b: 220, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:10, }"] [2025/04/04 20:23:51.557 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 221, b: 221, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:11, }"] [2025/04/04 20:23:51.664 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af] [captureID=b9674dd6-d9d0-4f0c-b073-745ef81fd58a] [processorDetail="{\"table_ids\":[116]}"] [2025/04/04 20:23:51.657 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 222, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:12, a: 222, }"] [2025/04/04 20:23:51.757 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 223, b: 223, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:13, }"] [2025/04/04 20:23:51.858 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 224, b: 224, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:14, }"] [2025/04/04 20:23:51.814 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af] [captureID=d65388e2-83a2-433b-aa64-9f7681136d23] [processorDetail="{\"table_ids\":[118]}"] [2025/04/04 20:23:51.814 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"9ae9b302-243a-495e-973f-1b3f806bd736\":[],\"b9674dd6-d9d0-4f0c-b073-745ef81fd58a\":[{\"ID\":116,\"Changefeed\":\"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af\"}],\"d65388e2-83a2-433b-aa64-9f7681136d23\":[{\"ID\":118,\"Changefeed\":\"5471bf6c-9bfe-4955-ab8c-9c5bb6a294af\"}]}"] [2025/04/04 20:23:51.814 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=5471bf6c-9bfe-4955-ab8c-9c5bb6a294af&target-cp-id=d65388e2-83a2-433b-aa64-9f7681136d23&table-id=116"] [2025/04/04 20:23:51.864 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=116] [2025/04/04 20:23:51.864 +08:00] [INFO] [main.go:113] ["all tables are moved"] [sourceCapture=b9674dd6-d9d0-4f0c-b073-745ef81fd58a] [targetCapture=d65388e2-83a2-433b-aa64-9f7681136d23] check diff successfully [2025/04/04 20:23:51.958 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:15, a: 225, b: 225, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:52.058 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 226, b: 226, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:16, }"] table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2025/04/04 20:23:52.158 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 227, b: 227, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:17, }"] [2025/04/04 20:23:52.258 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:18, a: 228, b: 228, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:52.358 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 229, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:19, a: 229, }"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14361.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.check2 exists [2025/04/04 20:23:52.459 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:20, a: 230, b: 230, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:52.559 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:21, a: 231, b: 231, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:52.659 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:22, a: 232, b: 232, }"] table batch_add_table.finish_mark not exists for 14-th check, retry later check diff successfully [2025/04/04 20:23:52.759 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 233, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:23, a: 233, }"] [2025/04/04 20:23:52.859 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:24, a: 234, b: 234, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] wait process 8300 exit for 1-th time... wait process 8300 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8300) - No such process wait process 8300 exit for 3-th time... process 8300 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 20:23:53 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.84628465.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:52.960 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 235, b: 235, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:25, }"] [2025/04/04 20:23:53.060 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 236, b: 236, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:26, }"] [2025/04/04 20:23:53.160 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:27, a: 237, b: 237, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d59e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118693985878022 MinTableBarrierTs:457118693985878024 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e835ebf-beb6-4eb1-af28-cd23447fe277 {"id":"1e835ebf-beb6-4eb1-af28-cd23447fe277","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769429} /tidb/cdc/default/__cdc_meta__/capture/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee {"id":"a226a456-03ea-4a17-bde9-f4d5c2e6a9ee","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89d45 a226a456-03ea-4a17-bde9-f4d5c2e6a9ee /tidb/cdc/default/__cdc_meta__/owner/22319600bee89dae 1e835ebf-beb6-4eb1-af28-cd23447fe277 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118693985878022,"min-table-barrier-ts":457118693985878024,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e835ebf-beb6-4eb1-af28-cd23447fe277/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d59e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118693985878022 MinTableBarrierTs:457118693985878024 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e835ebf-beb6-4eb1-af28-cd23447fe277 {"id":"1e835ebf-beb6-4eb1-af28-cd23447fe277","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769429} /tidb/cdc/default/__cdc_meta__/capture/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee {"id":"a226a456-03ea-4a17-bde9-f4d5c2e6a9ee","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89d45 a226a456-03ea-4a17-bde9-f4d5c2e6a9ee /tidb/cdc/default/__cdc_meta__/owner/22319600bee89dae 1e835ebf-beb6-4eb1-af28-cd23447fe277 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118693985878022,"min-table-barrier-ts":457118693985878024,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e835ebf-beb6-4eb1-af28+ grep -q 'failed to get info:' -cd23447fe277/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d59e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118693985878022 MinTableBarrierTs:457118693985878024 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e835ebf-beb6-4eb1-af28-cd23447fe277 {"id":"1e835ebf-beb6-4eb1-af28-cd23447fe277","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769429} /tidb/cdc/default/__cdc_meta__/capture/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee {"id":"a226a456-03ea-4a17-bde9-f4d5c2e6a9ee","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769423} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89d45 a226a456-03ea-4a17-bde9-f4d5c2e6a9ee /tidb/cdc/default/__cdc_meta__/owner/22319600bee89dae 1e835ebf-beb6-4eb1-af28-cd23447fe277 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118693985878022,"min-table-barrier-ts":457118693985878024,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e835ebf-beb6-4eb1-af28-cd23447fe277/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a226a456-03ea-4a17-bde9-f4d5c2e6a9ee/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "a226a456-03ea-4a17-bde9-f4d5c2e6a9ee" | grep id [2025/04/04 20:23:53.260 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 238, b: 238, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:28, }"] [2025/04/04 20:23:53.360 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 239, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:29, a: 239, }"] wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:53 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cd7d357-ec78-4b31-a7ff-26452a58f20d {"id":"6cd7d357-ec78-4b31-a7ff-26452a58f20d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1be34d5 6cd7d357-ec78-4b31-a7ff-26452a58f20d /tidb/cdc/default/default/upstream/7489432630677090175 {"id":7489432630677090175,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cd7d357-ec78-4b31-a7ff-26452a58f20d {"id":"6cd7d357-ec78-4b31-a7ff-26452a58f20d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1be34d5 6cd7d357-ec78-4b31-a7ff-26452a58f20d /tidb/cdc/default/default/upstream/7489432630677090175 {"id":7489432630677090175,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cd7d357-ec78-4b31-a7ff-26452a58f20d {"id":"6cd7d357-ec78-4b31-a7ff-26452a58f20d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1be34d5 6cd7d357-ec78-4b31-a7ff-26452a58f20d /tidb/cdc/default/default/upstream/7489432630677090175 {"id":7489432630677090175,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` [2025/04/04 20:23:53.460 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:30, a: 240, b: 240, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:23:53.561 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 241, b: 241, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:05, }"] [2025/04/04 20:23:53.661 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:32, a: 242, b: 242, }"] wait process cdc.test exit for 2-th time... check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:23:53.761 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 243, b: 243, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:33, }"] [2025/04/04 20:23:53.862 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 245, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:35, a: 245, }"] [2025/04/04 20:23:53.962 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:27:36, a: 246, b: 246, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 246, b: 246, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: J69KC5U1Ax, d: 20:27:36, }"] [2025/04/04 20:23:54.062 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:27:37, a: 247, b: 247, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 247, b: 247, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Ki9MBXgVoG, d: 20:27:37, }"] [2025/04/04 20:23:54.162 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:27:38, a: 248, b: 248, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 248, b: 248, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 6hXFyCf, d: 20:27:38, }"] "cluster-id": "default" "id": "1e835ebf-beb6-4eb1-af28-cd23447fe277", "cluster-id": "default" run task successfully table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:54 CST 2025] <<<<<< run test case move_table success! >>>>>> + set +x + tso='457118694155223043 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118694155223043 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:23:54 CST 2025] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1440014402.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:54.262 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 249, b: 249, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:39, }"] [row2="{ QH1PS: AHXOqSCprEzxyhG4N5tHh2spHPFGaWvbCJTne, c: x, d: 20:27:39, a: 249, b: 249, }"] [2025/04/04 20:23:54.362 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 250, b: 250, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:40, }"] pass test: owner apis pass test: set log level ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:23:54.463 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 251, b: 251, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:41, }"] [2025/04/04 20:23:54.563 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:04, a: 254, b: 254, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:54.663 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:45, a: 255, b: 255, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] table batch_add_table.finish_mark exists table test.t_1 does not exists check diff successfully [2025/04/04 20:23:54.763 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 256, b: 256, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:46, }"] [2025/04/04 20:23:54.863 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:47, a: 257, b: 257, }"] [row2="{ b: 257, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 7hUJT, d: 20:27:47, a: 257, }"] + cleanup_process cdc.test [2025/04/04 20:23:54.964 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:48, a: 258, b: 258, }"] [row2="{ a: 258, b: 258, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: mkh2v7Z, d: 20:27:48, }"] [2025/04/04 20:23:55.064 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:51, a: 261, b: 261, }"] [2025/04/04 20:23:55.164 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:27:52, a: 262, b: 262, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] wait process cdc.test exit for 1-th time... [2025/04/04 20:23:55.264 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 263, b: 263, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:53, }"] [2025/04/04 20:23:55.364 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 264, b: 264, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:54, }"] capture_id: 1e835ebf-beb6-4eb1-af28-cd23447fe277 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2 100 2 0 0 541 0 --:--:-- --:--:-- --:--:-- 666 {}cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep 1e835ebf-beb6-4eb1-af28-cd23447fe277 -A1 | grep '"is-owner": true' wait process cdc.test exit for 1-th time... [2025/04/04 20:23:55.464 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:27:55, a: 265, b: 265, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 265, b: 265, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: ipVuD08, d: 20:27:55, }"] [2025/04/04 20:23:55.565 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:27:56, a: 266, b: 266, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: FW, d: 20:27:56, a: 266, b: 266, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:23:55.665 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 267, b: 267, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:38, }"] [row2="{ b: 267, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IOajuljy, d: 20:40:38, a: 267, }"] changefeed is working normally rts: 457118694368084000->457118694761300000 checkpoint: 457118694368084000->457118694761300000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/04 20:23:55.182 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [duration=34.603069ms] [sql="CREATE DATABASE `ddl_reentrant`"] run task successfully wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... [2025/04/04 20:23:55.765 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 268, b: 268, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:27:58, }"] [row2="{ a: 268, b: 268, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: c, d: 20:27:58, }"] [2025/04/04 20:23:55.865 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:27:59, a: 269, b: 269, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 269, b: 269, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 3FpwZpR, d: 20:27:59, }"] [2025/04/04 20:23:55.965 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 270, b: 270, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:00, }"] table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later [2025/04/04 20:23:56.065 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 271, b: 271, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:01, }"] [2025/04/04 20:23:56.166 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 272, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:02, a: 272, }"] resume changefeed 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 from 457118694407405573 check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2e000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17822415-9455-4ad9-845a-b0be5b50af41 {"id":"17822415-9455-4ad9-845a-b0be5b50af41","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769433} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ebdc 17822415-9455-4ad9-845a-b0be5b50af41 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/17822415-9455-4ad9-845a-b0be5b50af41/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2e000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17822415-9455-4ad9-845a-b0be5b50af41 {"id":"17822415-9455-4ad9-845a-b0be5b50af41","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769433} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ebdc 17822415-9455-4ad9-845a-b0be5b50af41 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/17822415-9455-4ad9-845a-b0be5b50af41/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e2e000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17822415-9455-4ad9-845a-b0be5b50af41 {"id":"17822415-9455-4ad9-845a-b0be5b50af41","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769433} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ebdc 17822415-9455-4ad9-845a-b0be5b50af41 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/17822415-9455-4ad9-845a-b0be5b50af41/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit ++ date + echo '[Fri Apr 4 20:23:56 CST 2025] <<<<<< run test case api_v2 success! >>>>>>' [Fri Apr 4 20:23:56 CST 2025] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:23:56 CST 2025] <<<<<< run test case batch_add_table success! >>>>>> [2025/04/04 20:23:56.266 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:28:03, a: 273, b: 273, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:56.366 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:28:04, a: 274, b: 274, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:56.466 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 275, b: 275, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:05, }"] [2025/04/04 20:23:56.566 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 276, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:06, a: 276, }"] [row2="{ a: 276, b: 276, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IdPNyg9zA, d: 20:28:06, }"] [2025/04/04 20:23:56.666 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:28:07, a: 277, b: 277, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: aYaep, d: 20:28:07, a: 277, b: 277, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432662678631270 TiDB Cluster ID. tikv_gc_leader_uuid 6580307e5bc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:7557, start at 2025-04-04 20:23:54.49146943 +0800 CST m=+2.168574098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:54.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:54.479 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:54.479 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432662678631270 TiDB Cluster ID. tikv_gc_leader_uuid 6580307e5bc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:7557, start at 2025-04-04 20:23:54.49146943 +0800 CST m=+2.168574098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:54.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:54.479 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:54.479 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432672591216328 TiDB Cluster ID. tikv_gc_leader_uuid 6580307e8c8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-w-xqv7n, pid:7609, start at 2025-04-04 20:23:54.713857655 +0800 CST m=+2.319176492 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:25:54.721 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:23:54.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:13:54.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:23:56.767 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 278, b: 278, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:08, }"] [row2="{ d: 20:28:08, a: 278, b: 278, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: rFFHq8h, }"] [2025/04/04 20:23:56.867 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 279, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:09, a: 279, }"] [row2="{ c: wUJTW7, d: 20:28:09, a: 279, b: 279, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:23:56.967 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 280, b: 280, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:10, }"] [2025/04/04 20:23:57.067 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:28:11, a: 281, b: 281, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:23:57.167 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 282, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:12, a: 282, }"] [row2="{ a: 282, b: 282, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: buUqKkGutF, d: 20:28:12, }"] "is-owner": true, run task successfully test_resign_owner: pass [2025/04/04 20:23:57.267 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:13, a: 283, b: 283, }"] [row2="{ a: 283, b: 283, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: It, d: 20:28:13, }"] [2025/04/04 20:23:57.368 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:28:15, a: 285, b: 285, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: AL, d: 20:28:15, a: 285, b: 285, }"] [2025/04/04 20:23:57.468 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:16, a: 286, b: 286, }"] [row2="{ a: 286, b: 286, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: RVKSe, d: 20:28:16, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:23:57 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a33432a-b898-42e4-a2d6-c6a21f6e86e6 {"id":"6a33432a-b898-42e4-a2d6-c6a21f6e86e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769434} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1c04ee6 6a33432a-b898-42e4-a2d6-c6a21f6e86e6 /tidb/cdc/default/default/upstream/7489432636279251966 {"id":7489432636279251966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a33432a-b898-42e4-a2d6-c6a21f6e86e6 {"id":"6a33432a-b898-42e4-a2d6-c6a21f6e86e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769434} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1c04ee6 6a33432a-b898-42e4-a2d6-c6a21f6e86e6 /tidb/cdc/default/default/upstream/7489432636279251966 {"id":7489432636279251966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a33432a-b898-42e4-a2d6-c6a21f6e86e6 {"id":"6a33432a-b898-42e4-a2d6-c6a21f6e86e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769434} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1c04ee6 6a33432a-b898-42e4-a2d6-c6a21f6e86e6 /tidb/cdc/default/default/upstream/7489432636279251966 {"id":7489432636279251966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14463.out cli changefeed create --start-ts=457118694155223043 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 00ef58a5-20a1-4467-b1bc-7c82fa5320bd Info: {"upstream_id":7489432636279251966,"namespace":"default","id":"00ef58a5-20a1-4467-b1bc-7c82fa5320bd","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:23:57.61964548+08:00","start_ts":457118694155223043,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118694155223043,"checkpoint_ts":457118694155223043,"checkpoint_time":"2025-04-04 20:23:52.660"} PASS wait process cdc.test exit for 1-th time... [2025/04/04 20:23:57.568 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 287, b: 287, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:17, }"] [row2="{ d: 20:28:17, a: 287, b: 287, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: nDcA, }"] [2025/04/04 20:23:57.668 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:38, a: 288, b: 288, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 288, b: 288, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: yi3lozg, d: 20:30:38, }"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:23:57.768 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 289, b: 289, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:19, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: NfA, d: 20:28:19, a: 289, b: 289, }"] [2025/04/04 20:23:57.869 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 290, b: 290, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:22, }"] [2025/04/04 20:23:57.969 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:28:23, a: 291, b: 291, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] wait process cdc.test exit for 2-th time... run task failed 1-th time, retry later [2025/04/04 20:23:58.069 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 292, b: 292, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:16, }"] [2025/04/04 20:23:58.169 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 293, b: 293, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:25, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 0a, d: 20:28:25, a: 293, b: 293, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_kill_capture [Fri Apr 4 20:23:58 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.57355737.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:58.269 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 294, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:26, a: 294, }"] [row2="{ d: 20:28:26, a: 294, b: 294, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: oroom, }"] [2025/04/04 20:23:58.369 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 295, b: 295, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:27, }"] [row2="{ a: 295, b: 295, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: T8bnC1, d: 20:28:27, }"] [2025/04/04 20:23:58.470 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 296, b: 296, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:28, }"] [row2="{ a: 296, b: 296, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: i7vsj, d: 20:28:28, }"] table consistent_replicate_storage_file.check1 not exists for 14-th check, retry later [2025/04/04 20:23:58.570 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 297, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:29, a: 297, }"] [row2="{ d: 20:28:29, a: 297, b: 297, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: d, }"] [2025/04/04 20:23:58.670 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 298, b: 298, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:30, }"] [row2="{ b: 298, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: vHwuY, d: 20:28:30, a: 298, }"] [Fri Apr 4 20:23:58 CST 2025] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.88538855.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:23:58.770 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:28:31, a: 299, b: 299, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: L14E9cHGkF, d: 20:28:31, a: 299, b: 299, }"] [2025/04/04 20:23:58.870 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 300, b: 300, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:34, }"] [row2="{ b: 300, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: x1UC6jg, d: 20:28:34, a: 300, }"] [2025/04/04 20:23:58.970 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 301, b: 301, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:35, }"] [row2="{ c: mBmj, d: 20:28:35, a: 301, b: 301, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] + set +x table savepoint.finish_mark not exists for 1-th check, retry later [2025/04/04 20:23:59.071 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 302, b: 302, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:36, }"] [row2="{ a: 302, b: 302, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: G4W, d: 20:28:36, }"] [2025/04/04 20:23:59.171 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 303, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:37, a: 303, }"] [row2="{ a: 303, b: 303, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 6lkvkQjyh, d: 20:28:37, }"] [2025/04/04 20:23:59.271 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 305, b: 305, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:39, }"] [row2="{ c: mqdioVg, d: 20:28:39, a: 305, b: 305, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:23:59.371 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:28:40, a: 306, b: 306, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 306, b: 306, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: ow, d: 20:28:40, }"] [2025/04/04 20:23:59.471 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 307, b: 307, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:41, }"] [row2="{ a: 307, b: 307, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: s, d: 20:28:41, }"] [2025/04/04 20:23:59.572 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 308, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:42, a: 308, }"] [row2="{ a: 308, b: 308, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Va, d: 20:28:42, }"] [2025/04/04 20:23:59.672 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 309, b: 309, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:43, }"] [row2="{ b: 309, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: YKR, d: 20:28:43, a: 309, }"] [2025/04/04 20:23:59.772 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 310, b: 310, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:44, }"] [row2="{ d: 20:28:44, a: 310, b: 310, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Ijv3f9M11Y, }"] [2025/04/04 20:23:59.872 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 311, b: 311, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:45, }"] [row2="{ a: 311, b: 311, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Jb1G4v9t, d: 20:28:45, }"] [2025/04/04 20:23:59.972 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:28:46, a: 312, b: 312, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 312, b: 312, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: QsF0bL7, d: 20:28:46, }"] check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:24:00.072 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 313, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:47, a: 313, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: iPz, d: 20:28:47, a: 313, b: 313, }"] [2025/04/04 20:24:00.173 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:48, a: 314, b: 314, }"] [row2="{ a: 314, b: 314, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 11Y, d: 20:28:48, }"] [2025/04/04 20:24:00.273 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:28:49, a: 315, b: 315, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 315, b: 315, QH1PS: iGvPajdZIoJx9Pg6jy8XEpLmfYypNyWAOMSluZ5XsWC, c: ER6, d: 20:28:49, }"] table consistent_replicate_storage_file.check1 not exists for 15-th check, retry later [2025/04/04 20:24:00.373 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:50, a: 316, b: 316, }"] [row2="{ a: 316, b: 316, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: oiB4, d: 20:28:50, }"] [2025/04/04 20:24:00.473 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 317, b: 317, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:51, }"] [row2="{ a: 317, b: 317, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: odAmz, d: 20:28:51, }"] [2025/04/04 20:24:00.573 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:52, a: 318, b: 318, }"] [row2="{ a: 318, b: 318, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 5iaJ2Bj, d: 20:28:52, }"] [2025/04/04 20:24:00.674 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:28:53, a: 319, b: 319, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:28:53, a: 319, b: 319, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 1pyG16wT, }"] [2025/04/04 20:24:00.774 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 320, b: 320, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:54, }"] [row2="{ c: D, d: 20:28:54, a: 320, b: 320, QH1PS: purl9VTSp6IoMmGc2s, }"] [2025/04/04 20:24:00.874 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 321, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:55, a: 321, }"] [row2="{ b: 321, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 3E76kM1, d: 20:28:55, a: 321, }"] [2025/04/04 20:24:00.974 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:57, a: 323, b: 323, }"] [row2="{ a: 323, b: 323, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: zD9TAK4hI5, d: 20:28:57, }"] [2025/04/04 20:24:01.074 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 324, b: 324, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:58, }"] [row2="{ a: 324, b: 324, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: RNCdXNd, d: 20:28:58, }"] [2025/04/04 20:24:01.174 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:28:59, a: 325, b: 325, }"] [row2="{ a: 325, b: 325, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 7mOVN9T, d: 20:28:59, }"] [2025/04/04 20:24:01.275 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 326, b: 326, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:00, }"] [row2="{ c: KWxk, d: 20:29:00, a: 326, b: 326, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:01.375 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 327, b: 327, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:01, }"] [row2="{ a: 327, b: 327, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: n, d: 20:29:01, }"] [2025/04/04 20:24:01.475 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 328, b: 328, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:02, }"] [row2="{ a: 328, b: 328, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Lgr7, d: 20:29:02, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014e9830 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118696345436165 MinTableBarrierTs:457118696345436169 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118696345436169, checkpointTs: 457118696345436169, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118696345436165, checkpointTs: 457118696345436165, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118696358281217, checkpointTs: 457118696345436169, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f75639be-2501-49e8-8716-8eb9165e0dad {"id":"f75639be-2501-49e8-8716-8eb9165e0dad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e2f f75639be-2501-49e8-8716-8eb9165e0dad /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118696345436165,"min-table-barrier-ts":457118696345436169,"admin-job-type":0} /tidb/cdc/default/default/task/position/f75639be-2501-49e8-8716-8eb9165e0dad/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014e9830 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118696345436165 MinTableBarrierTs:457118696345436169 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118696345436169, checkpointTs: 457118696345436169, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118696345436165, checkpointTs: 457118696345436165, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118696358281217, checkpointTs: 457118696345436169, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f75639be-2501-49e8-8716-8eb9165e0dad {"id":"f75639be-2501-49e8-8716-8eb9165e0dad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e2f f75639be-2501-49e8-8716-8eb9165e0dad /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118696345+ grep -q 'failed to get info:' 436165,"min-table-barrier-ts":457118696345436169,"admin-job-type":0} /tidb/cdc/default/default/task/position/f75639be-2501-49e8-8716-8eb9165e0dad/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014e9830 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118696345436165 MinTableBarrierTs:457118696345436169 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118696345436169, checkpointTs: 457118696345436169, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118696345436165, checkpointTs: 457118696345436165, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118696358281217, checkpointTs: 457118696345436169, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f75639be-2501-49e8-8716-8eb9165e0dad {"id":"f75639be-2501-49e8-8716-8eb9165e0dad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e2f f75639be-2501-49e8-8716-8eb9165e0dad /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118696345+ grep -q 'etcd info' 436165,"min-table-barrier-ts":457118696345436169,"admin-job-type":0} /tidb/cdc/default/default/task/position/f75639be-2501-49e8-8716-8eb9165e0dad/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table savepoint.finish_mark not exists for 2-th check, retry later [2025/04/04 20:24:01.575 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 329, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:03, a: 329, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 72d, d: 20:29:03, a: 329, b: 329, }"] [2025/04/04 20:24:01.675 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:05, a: 330, b: 330, }"] [row2="{ a: 330, b: 330, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: qaxP32UjfU, d: 20:29:05, }"] [2025/04/04 20:24:01.775 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:08, a: 333, b: 333, }"] [row2="{ a: 333, b: 333, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Fgf, d: 20:29:08, }"] changefeed is working normally rts: 457118696072020000->457118696334164000 checkpoint: 457118696072020000->457118696334164000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2025/04/04 20:23:58.150 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457118694407405569] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [error="Error 1007 (HY000): Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:24:01.876 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:29:09, a: 334, b: 334, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 334, b: 334, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: ppwOTsb, d: 20:29:09, }"] [2025/04/04 20:24:01.976 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 335, b: 335, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:10, }"] [row2="{ a: 335, b: 335, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: kxLgh, d: 20:29:10, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:01 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c {"id":"e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769439} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1dce2d8 e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c /tidb/cdc/default/default/upstream/7489432662678631270 {"id":7489432662678631270,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c {"id":"e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769439} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1dce2d8 e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c /tidb/cdc/default/default/upstream/7489432662678631270 {"id":7489432662678631270,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c {"id":"e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769439} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1dce2d8 e4c1213a-81dd-4ceb-b50e-bc2e9ca83c5c /tidb/cdc/default/default/upstream/7489432662678631270 {"id":7489432662678631270,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2025/04/04 20:24:02.076 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 336, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:11, a: 336, }"] [row2="{ a: 336, b: 336, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: V7YGMi5fqC, d: 20:29:11, }"] [2025/04/04 20:24:02.176 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:29:12, a: 337, b: 337, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 337, b: 337, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: KXvC6BFECO, d: 20:29:12, }"] [2025/04/04 20:24:02.276 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 338, b: 338, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:13, }"] [row2="{ a: 338, b: 338, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: aFrgZRCg, d: 20:29:13, }"] check_changefeed_status 127.0.0.1:8300 879c10d3-9068-4283-b61b-a5623e496338 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=879c10d3-9068-4283-b61b-a5623e496338 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/879c10d3-9068-4283-b61b-a5623e496338/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 951 0 --:--:-- --:--:-- --:--:-- 945 100 86 100 86 0 0 950 0 --:--:-- --:--:-- --:--:-- 945 + info='{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' + echo '{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' {"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237} ++ echo '{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 879c10d3-9068-4283-b61b-a5623e496338 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=879c10d3-9068-4283-b61b-a5623e496338 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/879c10d3-9068-4283-b61b-a5623e496338/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table consistent_replicate_storage_file.check1 not exists for 16-th check, retry later [2025/04/04 20:24:02.377 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:14, a: 339, b: 339, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 339, b: 339, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 5eei, d: 20:29:14, }"] [2025/04/04 20:24:02.477 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:15, a: 340, b: 340, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: HAS7R, d: 20:29:15, a: 340, b: 340, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1078 0 --:--:-- --:--:-- --:--:-- 1088 + info='{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' + echo '{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' {"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237} ++ echo '{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":457118696558559237,"checkpoint_ts":457118696558559237}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later [2025/04/04 20:24:02.577 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:29:16, a: 341, b: 341, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:29:16, a: 341, b: 341, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: EnjBJVXR, }"] [2025/04/04 20:24:02.677 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 342, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:17, a: 342, }"] [row2="{ a: 342, b: 342, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: cL68FoXZ, d: 20:29:17, }"] [2025/04/04 20:24:02.777 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:18, a: 343, b: 343, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 57iJlDp, d: 20:29:18, a: 343, b: 343, }"] [2025/04/04 20:24:02.878 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:19, a: 344, b: 344, }"] [row2="{ c: XUAz7J2e, d: 20:29:19, a: 344, b: 344, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:02.978 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 345, b: 345, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:20, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: CeQ2gJH, d: 20:29:20, a: 345, b: 345, }"] "is-owner": true, run task successfully [2025/04/04 20:24:03.078 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 346, b: 346, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:21, }"] [row2="{ a: 346, b: 346, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: M0pS7Cdx, d: 20:29:21, }"] [2025/04/04 20:24:03.178 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 347, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:22, a: 347, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 3cKmEQ5xC, d: 20:29:22, a: 347, b: 347, }"] [2025/04/04 20:24:03.278 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 349, b: 349, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:24, }"] [row2="{ b: 349, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: hk9MA4hq5p, d: 20:29:24, a: 349, }"] [2025/04/04 20:24:03.378 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:25, a: 350, b: 350, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: bJN5Y7P00q, d: 20:29:25, a: 350, b: 350, QH1PS: 5vlSU2UUzPjCHQZgzbpBMnN9yLAVQDOOPIgtnO, }"] [2025/04/04 20:24:03.479 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:26, a: 351, b: 351, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 351, b: 351, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Dh4bnj, d: 20:29:26, }"] table savepoint.finish_mark exists check diff successfully [2025/04/04 20:24:03.579 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 352, b: 352, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:27, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: MjMoscYSLa, d: 20:29:27, a: 352, b: 352, }"] [2025/04/04 20:24:03.679 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 353, b: 353, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:28, }"] [row2="{ a: 353, b: 353, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: idv3YYG, d: 20:29:28, }"] [2025/04/04 20:24:03.779 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 354, b: 354, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:29, }"] [row2="{ a: 354, b: 354, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: oNEMxTsgR, d: 20:29:29, }"] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 26473 0 --:--:-- --:--:-- --:--:-- 27625 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 20:23:50.615","puller_resolved_ts":"2025-04-04 20:23:44.566","last_synced_ts":"2025-04-04 20:21:35.166","now_ts":"2025-04-04 20:23:52.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:23:50.615","puller_resolved_ts":"2025-04-04' '20:23:44.566","last_synced_ts":"2025-04-04' '20:21:35.166","now_ts":"2025-04-04' '20:23:52.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9280 10.4 0.0 14092440 154416 ? Sl 20:21 0:16 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9351 7.5 0.0 14091672 143464 ? Sl 20:21 0:11 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ awk '{print $2}' ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1743769439.0218463,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004f70a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769439.0219326,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769439.0953708,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041b8e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743769439.095454,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769439.1103945,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0030a4c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769439.110462,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} wait process cdc.test exit for 1-th time... [2025/04/04 20:24:03.880 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:30, a: 355, b: 355, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 355, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: caIhSXf, d: 20:29:30, a: 355, }"] [2025/04/04 20:24:03.980 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 356, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:31, a: 356, }"] [row2="{ a: 356, b: 356, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: VMt, d: 20:29:31, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 17-th check, retry later [2025/04/04 20:24:04.080 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:32, a: 357, b: 357, }"] [row2="{ a: 357, b: 357, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: RdhPtN, d: 20:29:32, }"] [2025/04/04 20:24:04.180 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:33, a: 358, b: 358, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 358, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: aMQxj, d: 20:29:33, a: 358, }"] [2025/04/04 20:24:04.280 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 359, b: 359, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:34, }"] [row2="{ a: 359, b: 359, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: VzQZl44Oe, d: 20:29:34, }"] wait process cdc.test exit for 2-th time... table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later run task failed 1-th time, retry later [2025/04/04 20:24:04.381 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 360, b: 360, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:35, }"] [row2="{ c: EwF, d: 20:29:35, a: 360, b: 360, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:04.481 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:36, a: 361, b: 361, }"] [row2="{ c: vqEQK, d: 20:29:36, a: 361, b: 361, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:04.581 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 362, b: 362, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:37, }"] [row2="{ a: 362, b: 362, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: vOqEu0cwD, d: 20:29:37, }"] [2025/04/04 20:24:04.681 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 363, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:38, a: 363, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: qq4rYKnZJ, d: 20:29:38, a: 363, b: 363, }"] [2025/04/04 20:24:04.781 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 364, b: 364, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:39, }"] [row2="{ a: 364, b: 364, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 3ZeyP, d: 20:29:39, }"] owner pid: 5740 owner id f75639be-2501-49e8-8716-8eb9165e0dad table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:24:04 CST 2025] <<<<<< run test case savepoint success! >>>>>> [2025/04/04 20:24:04.881 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 365, b: 365, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:40, }"] [row2="{ d: 20:29:40, a: 365, b: 365, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: zeD6, }"] [2025/04/04 20:24:04.982 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 366, b: 366, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:41, }"] [row2="{ a: 366, b: 366, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: WRifUDr, d: 20:29:41, }"] [2025/04/04 20:24:05.082 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:29:42, a: 367, b: 367, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 367, b: 367, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: TCLGHe7WaN, d: 20:29:42, }"] [2025/04/04 20:24:05.182 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 368, b: 368, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:43, }"] [row2="{ a: 368, b: 368, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IkZbJdjAjS, d: 20:29:43, }"] [2025/04/04 20:24:05.282 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:44, a: 369, b: 369, }"] [row2="{ a: 369, b: 369, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Sa, d: 20:29:44, }"] [2025/04/04 20:24:05.382 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 370, b: 370, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:45, }"] [row2="{ a: 370, b: 370, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 2e, d: 20:29:45, }"] [2025/04/04 20:24:05.483 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 371, b: 371, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:46, }"] [row2="{ a: 371, b: 371, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: yaFV, d: 20:29:46, }"] [2025/04/04 20:24:05.583 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 372, b: 372, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:47, }"] [row2="{ a: 372, b: 372, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 8, d: 20:29:47, }"] [2025/04/04 20:24:05.683 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 373, b: 373, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:48, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: PmZcNpiSZ, d: 20:29:48, a: 373, b: 373, }"] [2025/04/04 20:24:05.783 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 374, b: 374, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:31, }"] [row2="{ a: 374, b: 374, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 5, d: 20:37:31, }"] table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later [2025/04/04 20:24:05.883 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 375, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:50, a: 375, }"] [row2="{ d: 20:29:50, a: 375, b: 375, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: AcP9L, }"] [2025/04/04 20:24:05.983 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 376, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:51, a: 376, }"] [row2="{ a: 376, b: 376, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: dEoeYMAOI, d: 20:29:51, }"] [2025/04/04 20:24:06.084 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 377, b: 377, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:52, }"] [row2="{ a: 377, b: 377, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: c7, d: 20:29:52, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process 8469 exit for 1-th time... wait process 8469 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8469) - No such process wait process 8469 exit for 3-th time... process 8469 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 20:24:05 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.86008602.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:06.184 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 378, b: 378, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:53, }"] [row2="{ a: 378, b: 378, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: VHBxqXKznT, d: 20:29:53, }"] [2025/04/04 20:24:06.284 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 379, b: 379, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:54, }"] [row2="{ b: 379, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: RJ2kBj, d: 20:29:54, a: 379, }"] check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:24:06.384 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 380, b: 380, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:55, }"] [row2="{ a: 380, b: 380, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: JWS7y, d: 20:29:55, }"] [2025/04/04 20:24:06.484 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 381, b: 381, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:56, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 7DK9g4SZ5C, d: 20:29:56, a: 381, b: 381, }"] [2025/04/04 20:24:06.585 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:29:57, a: 382, b: 382, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:29:57, a: 382, b: 382, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: ftdTz, }"] Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Fri Apr 4 20:24:06 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.58865888.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:06.685 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:29:58, a: 383, b: 383, }"] [row2="{ d: 20:29:58, a: 383, b: 383, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: gOAmlRm, }"] [2025/04/04 20:24:06.785 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 384, b: 384, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:40, }"] [row2="{ a: 384, b: 384, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: G2rMeNfA4L, d: 20:31:40, }"] table sink_hang.t2 exists check diff failed 1-th time, retry later pass check, checkpoint tso not forward after 10s run task successfully [2025/04/04 20:24:06.885 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 385, b: 385, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:00, }"] [row2="{ a: 385, b: 385, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: N0, d: 20:30:00, }"] [2025/04/04 20:24:06.985 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 386, b: 386, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:01, }"] [row2="{ b: 386, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: f9u, d: 20:30:01, a: 386, }"] [2025/04/04 20:24:07.085 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:30:02, a: 387, b: 387, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 387, b: 387, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 0QVys, d: 20:30:02, }"] {"level":"warn","ts":"2025-04-04T20:24:06.425578+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:06.427503+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:06.517514+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process 8281 exit for 1-th time... [2025/04/04 20:24:07.186 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 388, b: 388, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:03, }"] [row2="{ a: 388, b: 388, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: dOC, d: 20:30:03, }"] [2025/04/04 20:24:07.286 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:04, a: 389, b: 389, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: OM0SQg8X7j, d: 20:30:04, a: 389, b: 389, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:24:07.386 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 390, b: 390, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:05, }"] [row2="{ d: 20:30:05, a: 390, b: 390, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 4pljpzh, }"] [2025/04/04 20:24:07.486 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 391, b: 391, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:06, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 1hV, d: 20:30:06, a: 391, b: 391, }"] [2025/04/04 20:24:07.586 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:07, a: 392, b: 392, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 392, b: 392, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: uRasnksnem, d: 20:30:07, }"] [2025/04/04 20:24:07.687 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 393, b: 393, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:08, }"] [row2="{ a: 393, b: 393, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: hcr8CE, d: 20:30:08, }"] [2025/04/04 20:24:07.787 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 394, b: 394, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:09, }"] [row2="{ a: 394, b: 394, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: THeJbQfm6N, d: 20:30:09, }"] wait process 8281 exit for 2-th time... [2025/04/04 20:24:07.887 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 395, b: 395, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:10, }"] [row2="{ a: 395, b: 395, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: YD3llLA3E, d: 20:30:10, }"] [2025/04/04 20:24:07.987 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 396, b: 396, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:11, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: YN9kKbEL, d: 20:30:11, a: 396, b: 396, }"] [2025/04/04 20:24:08.087 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 397, b: 397, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:12, }"] [row2="{ a: 397, b: 397, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: HxOsg, d: 20:30:12, }"] changefeed is working normally rts: 457118697395847200->457118697920135200 checkpoint: 457118697657991200->457118697920135200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/04 20:24:02.129 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [duration=29.212509ms] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] run task successfully [2025/04/04 20:24:08.188 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:13, a: 398, b: 398, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:30:13, a: 398, b: 398, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: VI2, }"] [2025/04/04 20:24:08.288 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 399, b: 399, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:14, }"] [row2="{ a: 399, b: 399, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: v, d: 20:30:14, }"] [2025/04/04 20:24:08.388 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 400, b: 400, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:18, }"] [row2="{ a: 400, b: 400, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 1ilWGaLq, d: 20:30:18, }"] wait process 8281 exit for 3-th time... resume changefeed 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 from 457118696583200789 check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later [2025/04/04 20:24:08.488 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:19, a: 401, b: 401, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 401, b: 401, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 6WuVrWhb9, d: 20:30:19, }"] [2025/04/04 20:24:08.588 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:20, a: 402, b: 402, }"] [row2="{ a: 402, b: 402, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: c3, d: 20:30:20, }"] check diff failed 2-th time, retry later [2025/04/04 20:24:08.688 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 403, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:21, a: 403, }"] [row2="{ a: 403, b: 403, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IQnW5lCKB, d: 20:30:21, }"] [2025/04/04 20:24:08.789 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:22, a: 404, b: 404, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 404, b: 404, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 72WOE98G, d: 20:30:22, }"] [2025/04/04 20:24:08.889 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 406, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:24, a: 406, }"] [row2="{ a: 406, b: 406, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: jSTHMu5vMF, d: 20:30:24, }"] wait process 8281 exit for 4-th time... [2025/04/04 20:24:08.989 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 409, b: 409, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:27, }"] [row2="{ a: 409, b: 409, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: tVf, d: 20:30:27, }"] [2025/04/04 20:24:09.089 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:28, a: 410, b: 410, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 410, b: 410, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Pl, d: 20:30:28, }"] [2025/04/04 20:24:09.189 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 411, b: 411, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:29, }"] [row2="{ d: 20:30:29, a: 411, b: 411, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 8D5HdI6, }"] [2025/04/04 20:24:09.289 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 412, b: 412, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:30, }"] [row2="{ a: 412, b: 412, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 8GgT, d: 20:30:30, }"] [2025/04/04 20:24:09.390 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:30:31, a: 413, b: 413, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: mH, d: 20:30:31, a: 413, b: 413, }"] start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015c4c60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee71f518-d211-4699-a6a4-85f6a25f8e28 {"id":"ee71f518-d211-4699-a6a4-85f6a25f8e28","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769446} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ec6a ee71f518-d211-4699-a6a4-85f6a25f8e28 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee71f518-d211-4699-a6a4-85f6a25f8e28/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015c4c60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee71f518-d211-4699-a6a4-85f6a25f8e28 {"id":"ee71f518-d211-4699-a6a4-85f6a25f8e28","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769446} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ec6a ee71f518-d211-4699-a6a4-85f6a25f8e28 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee71f518-d211-4699-a6a4-85f6a25f8e28/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015c4c60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee71f518-d211-4699-a6a4-85f6a25f8e28 {"id":"ee71f518-d211-4699-a6a4-85f6a25f8e28","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769446} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ec6a ee71f518-d211-4699-a6a4-85f6a25f8e28 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee71f518-d211-4699-a6a4-85f6a25f8e28/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8281) - No such process wait process 8281 exit for 5-th time... process 8281 already exit [Fri Apr 4 20:24:09 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.88638865.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:09.490 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:32, a: 414, b: 414, }"] [row2="{ a: 414, b: 414, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: KYIGJdYW, d: 20:30:32, }"] [2025/04/04 20:24:09.590 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 415, b: 415, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:33, }"] [row2="{ c: 5, d: 20:30:33, a: 415, b: 415, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:09.690 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 416, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:34, a: 416, }"] [row2="{ b: 416, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: xA5sD0, d: 20:30:34, a: 416, }"] [2025/04/04 20:24:09.790 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 417, b: 417, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:35, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: grK7jPTiXk, d: 20:30:35, a: 417, b: 417, }"] [2025/04/04 20:24:09.891 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:36, a: 418, b: 418, }"] [row2="{ a: 418, b: 418, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: jxXD4, d: 20:30:36, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004226750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118698455695361 MinTableBarrierTs:457118698455695364 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60624ff1-8d8a-43f3-beb6-b5a425b14f93 {"id":"60624ff1-8d8a-43f3-beb6-b5a425b14f93","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769446} /tidb/cdc/default/__cdc_meta__/capture/f75639be-2501-49e8-8716-8eb9165e0dad {"id":"f75639be-2501-49e8-8716-8eb9165e0dad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e2f f75639be-2501-49e8-8716-8eb9165e0dad /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e98 60624ff1-8d8a-43f3-beb6-b5a425b14f93 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118698455695361,"min-table-barrier-ts":457118698455695364,"admin-job-type":0} /tidb/cdc/default/default/task/position/60624ff1-8d8a-43f3-beb6-b5a425b14f93/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f75639be-2501-49e8-8716-8eb9165e0dad/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004226750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118698455695361 MinTableBarrierTs:457118698455695364 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60624ff1-8d8a-43f3-beb6-b5a425b14f93 {"id":"60624ff1-8d8a-43f3-beb6-b5a425b14f93","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769446} /tidb/cdc/default/__cdc_meta__/capture/f75639be-2501-49e8-8716-8eb9165e0dad {"id":"f75639be-2501-49e8-8716-8eb9165e0dad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e2f f75639be-2501-49e8-8716-8eb9165e0dad /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e98 60624ff1-8d8a-43f3-beb6-b5a425b14f93 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118698455695361,"min-table-barrier-ts":457118698455695364,"admin-job-type":0} /tidb/cdc/default/default/task/position/60624ff1-8d8a-43f3-beb6-b5a425b14f93/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f75639be-2501-49e8-8716-8eb9165e0dad/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004226750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118698455695361 MinTableBarrierTs:457118698455695364 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60624ff1-8d8a-43f3-beb6-b5a425b14f93 {"id":"60624ff1-8d8a-43f3-beb6-b5a425b14f93","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769446} /tidb/cdc/default/__cdc_meta__/capture/f75639be-2501-49e8-8716-8eb9165e0dad {"id":"f75639be-2501-49e8-8716-8eb9165e0dad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769438} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e2f f75639be-2501-49e8-8716-8eb9165e0dad /tidb/cdc/default/__cdc_meta__/owner/22319600bee89e98 60624ff1-8d8a-43f3-beb6-b5a425b14f93 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118698455695361,"min-table-barrier-ts":457118698455695364,"admin-job-type":0} /tidb/cdc/default/default/task/position/60624ff1-8d8a-43f3-beb6-b5a425b14f93/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f75639be-2501-49e8-8716-8eb9165e0dad/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "f75639be-2501-49e8-8716-8eb9165e0dad" | grep id [2025/04/04 20:24:09.991 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:37, a: 419, b: 419, }"] [row2="{ a: 419, b: 419, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 81, d: 20:30:37, }"] [2025/04/04 20:24:10.091 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 420, b: 420, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:39, }"] [row2="{ d: 20:30:39, a: 420, b: 420, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: OZGnQKNBG7, }"] [2025/04/04 20:24:10.191 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 421, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:40, a: 421, }"] [row2="{ b: 421, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Z3riZl, d: 20:30:40, a: 421, }"] [2025/04/04 20:24:10.291 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 422, b: 422, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:41, }"] [row2="{ d: 20:30:41, a: 422, b: 422, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: t, }"] [2025/04/04 20:24:10.391 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 423, b: 423, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:42, }"] [row2="{ c: J9iA2zQeiQ, d: 20:30:42, a: 423, b: 423, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] run task failed 1-th time, retry later table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:24:10.491 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 424, b: 424, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:43, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Wv, d: 20:30:43, a: 424, b: 424, }"] [2025/04/04 20:24:10.592 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 425, b: 425, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:44, }"] [row2="{ a: 425, b: 425, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 72li1FFw2, d: 20:30:44, }"] [2025/04/04 20:24:10.692 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:30:45, a: 426, b: 426, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 426, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Y, d: 20:30:45, a: 426, }"] [2025/04/04 20:24:10.792 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 427, b: 427, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:46, }"] [row2="{ d: 20:30:46, a: 427, b: 427, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: bFnvpnSyn, }"] [2025/04/04 20:24:10.892 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 428, b: 428, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:47, }"] [row2="{ c: ZG, d: 20:30:47, a: 428, b: 428, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:24:10.992 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 429, b: 429, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:48, }"] [row2="{ c: 23, d: 20:30:48, a: 429, b: 429, QH1PS: WyuRlw73UagIg9LNHWkj9JAhtx, }"] [2025/04/04 20:24:11.093 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 430, b: 430, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:49, }"] [row2="{ a: 430, b: 430, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: q, d: 20:30:49, }"] check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used [2025/04/04 20:24:11.193 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 431, b: 431, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:50, }"] [row2="{ a: 431, b: 431, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: ysXkJ, d: 20:30:50, }"] [2025/04/04 20:24:11.293 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 432, b: 432, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:51, }"] [row2="{ d: 20:30:51, a: 432, b: 432, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: tX2n8rbLqo, }"] [2025/04/04 20:24:11.393 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 433, b: 433, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:52, }"] [row2="{ d: 20:30:52, a: 433, b: 433, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IGRKNNn, }"] "cluster-id": "default" "id": "60624ff1-8d8a-43f3-beb6-b5a425b14f93", "cluster-id": "default" run task successfully [2025/04/04 20:24:11.493 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 434, b: 434, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:53, }"] [row2="{ a: 434, b: 434, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: bMB9RsN, d: 20:30:53, }"] [2025/04/04 20:24:11.593 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 435, b: 435, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:54, }"] [row2="{ a: 435, b: 435, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: jDji3q7cu, d: 20:30:54, }"] wait process cdc.test exit for 2-th time... [2025/04/04 20:24:11.694 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 436, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:55, a: 436, }"] [row2="{ a: 436, b: 436, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: tYH, d: 20:30:55, }"] [2025/04/04 20:24:11.794 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:30:56, a: 437, b: 437, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:30:56, a: 437, b: 437, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: tvfHZDalj3, }"] [2025/04/04 20:24:11.894 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:57, a: 438, b: 438, }"] [row2="{ d: 20:30:57, a: 438, b: 438, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 8j, }"] [2025/04/04 20:24:11.994 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:30:58, a: 439, b: 439, }"] [row2="{ b: 439, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Ry, d: 20:30:58, a: 439, }"] [2025/04/04 20:24:12.094 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:30:59, a: 440, b: 440, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:30:59, a: 440, b: 440, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Vc, }"] table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 {"level":"warn","ts":1743769452.1096377,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004f70a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":1743769452.1800823,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041b8e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:24:12.195 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:31:00, a: 441, b: 441, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 441, b: 441, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 8A8T, d: 20:31:00, }"] [2025/04/04 20:24:12.295 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 442, b: 442, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:01, }"] [row2="{ a: 442, b: 442, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 2o7UTAg5YE, d: 20:31:01, }"] [2025/04/04 20:24:12.395 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 443, b: 443, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:02, }"] [row2="{ b: 443, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: mZrYUaQ6, d: 20:31:02, a: 443, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:24:12 CST 2025] <<<<<< run test case sink_hang success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:12.427176+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:12.459178+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:12.518703+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:24:12.495 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 444, b: 444, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:03, }"] [row2="{ a: 444, b: 444, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Bjp4ej5F, d: 20:31:03, }"] [2025/04/04 20:24:12.595 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:04, a: 445, b: 445, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: q2L4, d: 20:31:04, a: 445, b: 445, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:12.695 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 446, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:05, a: 446, }"] [row2="{ a: 446, b: 446, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: KHjQ, d: 20:31:05, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489432419196392627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:03.03501263 +0800 CST StartTs:457118681129025538 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00108eab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118681141870594} {CheckpointTs:457118685008756741 MinTableBarrierTs:457118699046305794 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/963e7fea-8895-423b-ae22-f1862de16200 {"id":"963e7fea-8895-423b-ae22-f1862de16200","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769449} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e88a 963e7fea-8895-423b-ae22-f1862de16200 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489432419196392627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:23:03.03501263+08:00","start-ts":457118681129025538,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118681141870594} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457118685008756741,"min-table-barrier-ts":457118699046305794,"admin-job-type":0} /tidb/cdc/default/default/task/position/963e7fea-8895-423b-ae22-f1862de16200/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489432419196392627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:03.03501263 +0800 CST StartTs:457118681129025538 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00108eab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118681141870594} {CheckpointTs:457118685008756741 MinTableBarrierTs:457118699046305794 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/963e7fea-8895-423b-ae22-f1862de16200 {"id":"963e7fea-8895-423b-ae22-f1862de16200","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769449} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e88a 963e7fea-8895-423b-ae22-f1862de16200 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489432419196392627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:23:03.03501263+08:00","start-ts":457118681129025538,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118681141870594} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457118685008756741,"min-table-barrier-ts":457118699046305794,"admin-job-type":0} /tidb/cdc/default/default/task/position/963e7fea-8895-423b-ae22-f1862de16200/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489432419196392627 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:03.03501263 +0800 CST StartTs:457118681129025538 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00108eab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118681141870594} {CheckpointTs:457118685008756741 MinTableBarrierTs:457118699046305794 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118685008756741, checkpointTs: 457118685008756741, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/963e7fea-8895-423b-ae22-f1862de16200 {"id":"963e7fea-8895-423b-ae22-f1862de16200","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769449} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c0f9e88a 963e7fea-8895-423b-ae22-f1862de16200 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489432419196392627,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-04T20:23:03.03501263+08:00","start-ts":457118681129025538,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118681141870594} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457118685008756741,"min-table-barrier-ts":457118699046305794,"admin-job-type":0} /tidb/cdc/default/default/task/position/963e7fea-8895-423b-ae22-f1862de16200/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432419196392627 {"id":7489432419196392627,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/04 20:24:12.795 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 447, b: 447, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:06, }"] [row2="{ a: 447, b: 447, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: uiCGqGF, d: 20:31:06, }"] [2025/04/04 20:24:12.896 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 448, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:07, a: 448, }"] [row2="{ b: 448, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: SyEKtV5EA, d: 20:31:07, a: 448, }"] [2025/04/04 20:24:12.996 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:31:08, a: 449, b: 449, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 449, b: 449, QH1PS: 4qlaLCZUw, c: JCccEXU5Bd, d: 20:31:08, }"] [2025/04/04 20:24:13.096 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 450, b: 450, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:09, }"] [row2="{ a: 450, b: 450, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: F, d: 20:31:09, }"] [2025/04/04 20:24:13.196 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 451, b: 451, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:10, }"] [row2="{ a: 451, b: 451, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: j1booPqE2, d: 20:31:10, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore [2025/04/04 20:24:13.296 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:31:11, a: 452, b: 452, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 452, b: 452, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: pspxzO, d: 20:31:11, }"] [2025/04/04 20:24:13.396 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 454, b: 454, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:13, }"] [row2="{ a: 454, b: 454, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: TF, d: 20:31:13, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:13.497 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:14, a: 455, b: 455, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: eW0, d: 20:31:14, a: 455, b: 455, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:13.597 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:16, a: 457, b: 457, }"] [row2="{ a: 457, b: 457, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: lopfm7pX98, d: 20:31:16, }"] [2025/04/04 20:24:13.697 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 458, b: 458, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:17, }"] [row2="{ a: 458, b: 458, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: KCKXx, d: 20:31:17, }"] [2025/04/04 20:24:13.797 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 459, b: 459, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:18, }"] [row2="{ a: 459, b: 459, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 5, d: 20:31:18, }"] [2025/04/04 20:24:13.898 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:20, a: 460, b: 460, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: gWpaDihf, d: 20:31:20, a: 460, b: 460, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:13.998 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 461, b: 461, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:21, }"] [row2="{ b: 461, QH1PS: pNlvoE4oK9IpKo, c: lSGPUcd3, d: 20:31:21, a: 461, }"] [2025/04/04 20:24:14.098 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:22, a: 462, b: 462, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 462, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: d7g8, d: 20:31:22, a: 462, }"] [2025/04/04 20:24:14.199 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 463, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:23, a: 463, }"] [row2="{ c: 0gS6, d: 20:31:23, a: 463, b: 463, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later changefeed is working normally rts: 457118699230855200->457118699492999200 checkpoint: 457118699230855200->457118699492999200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2025/04/04 20:24:09.161 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457118696583200785] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [error="Error 1050 (42S01): Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:24:14.299 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 464, b: 464, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:24, }"] [row2="{ c: vK9jAgvSB, d: 20:31:24, a: 464, b: 464, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:14.399 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 465, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:25, a: 465, }"] [row2="{ a: 465, b: 465, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: xji6Od, d: 20:31:25, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 changefeed is working normally rts: 457118685008756740->457118699570855940 checkpoint: 457118685008756740->457118699570855940 run task successfully nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2025/04/04 20:24:14.499 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 466, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:26, a: 466, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: W7C, d: 20:31:26, a: 466, b: 466, }"] [2025/04/04 20:24:14.599 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:27, a: 467, b: 467, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 22EleI, d: 20:31:27, a: 467, b: 467, }"] [2025/04/04 20:24:14.700 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 468, b: 468, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:28, }"] [row2="{ a: 468, b: 468, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 0TLwu, d: 20:31:28, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... [2025/04/04 20:24:14.800 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:31:29, a: 469, b: 469, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 469, b: 469, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: wcQdSWh2, d: 20:31:29, }"] [2025/04/04 20:24:14.900 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 470, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:30, a: 470, }"] [row2="{ a: 470, b: 470, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: y1uXC1inSz, d: 20:31:30, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:15.000 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 471, b: 471, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:31, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: c3mm, d: 20:31:31, a: 471, b: 471, }"] [2025/04/04 20:24:15.100 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:31:32, a: 472, b: 472, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 472, b: 472, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: NNjenMIS1, d: 20:31:32, }"] [2025/04/04 20:24:15.200 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:33, a: 473, b: 473, }"] [row2="{ a: 473, b: 473, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: cM3g, d: 20:31:33, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 [2025/04/04 20:24:15.301 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 474, b: 474, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:34, }"] [row2="{ a: 474, b: 474, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: gU2yW, d: 20:31:34, }"] [2025/04/04 20:24:15.401 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:35, a: 475, b: 475, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 475, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: e4, d: 20:31:35, a: 475, }"] wait process cdc.test exit for 2-th time... [2025/04/04 20:24:15.501 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 476, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:36, a: 476, }"] [row2="{ d: 20:31:36, a: 476, b: 476, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: F7XVEH, }"] [2025/04/04 20:24:15.601 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:37, a: 477, b: 477, }"] [row2="{ c: rdBQsgPn, d: 20:31:37, a: 477, b: 477, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:15.701 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 478, b: 478, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:38, }"] [row2="{ a: 478, b: 478, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: C5lj6QWz, d: 20:31:38, }"] [2025/04/04 20:24:15.801 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:39, a: 479, b: 479, }"] [row2="{ a: 479, b: 479, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: H, d: 20:31:39, }"] [2025/04/04 20:24:15.902 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 480, b: 480, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:41, }"] [row2="{ a: 480, b: 480, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: dZHQF, d: 20:31:41, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:24:15 CST 2025] <<<<<< run test case ddl_only_block_related_table success! >>>>>> run task failed 1-th time, retry later [2025/04/04 20:24:16.002 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 481, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:42, a: 481, }"] [row2="{ a: 481, b: 481, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: rY, d: 20:31:42, }"] [2025/04/04 20:24:16.102 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:43, a: 482, b: 482, }"] [row2="{ d: 20:31:43, a: 482, b: 482, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Ar8b9s, }"] [2025/04/04 20:24:16.202 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 483, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:44, a: 483, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Jos7V7W8m, d: 20:31:44, a: 483, b: 483, }"] [2025/04/04 20:24:16.302 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 484, b: 484, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:45, }"] [row2="{ a: 484, b: 484, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: C6wr, d: 20:31:45, }"] [2025/04/04 20:24:16.402 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:46, a: 485, b: 485, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 485, b: 485, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: UdQU8bEudf, d: 20:31:46, }"] [2025/04/04 20:24:16.503 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:31:47, a: 486, b: 486, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:31:47, a: 486, b: 486, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: w, }"] table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later [2025/04/04 20:24:16.603 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 487, b: 487, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:48, }"] [row2="{ d: 20:31:48, a: 487, b: 487, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 4pCpr, }"] [2025/04/04 20:24:16.703 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 488, b: 488, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:49, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: dmrYS, d: 20:31:49, a: 488, b: 488, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432759072875547 TiDB Cluster ID. tikv_gc_leader_uuid 658030922300007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:30171, start at 2025-04-04 20:24:14.738685668 +0800 CST m=+2.297375093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:14.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:14.732 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:14.732 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432759072875547 TiDB Cluster ID. tikv_gc_leader_uuid 658030922300007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:30171, start at 2025-04-04 20:24:14.738685668 +0800 CST m=+2.297375093 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:14.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:14.732 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:14.732 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432753241678650 TiDB Cluster ID. tikv_gc_leader_uuid 658030923bc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:30240, start at 2025-04-04 20:24:14.852083056 +0800 CST m=+2.350615779 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:14.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:14.831 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:14.831 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:24:16.803 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 489, b: 489, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:50, }"] [row2="{ b: 489, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: cj52hZ, d: 20:31:50, a: 489, }"] [2025/04/04 20:24:16.903 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:52, a: 490, b: 490, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 490, b: 490, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: elNNc2, d: 20:31:52, }"] [2025/04/04 20:24:17.003 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:53, a: 491, b: 491, }"] [row2="{ d: 20:31:53, a: 491, b: 491, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: aYdidMG2s, }"] + '[' mysql == mysql ']' + check_table_exists test.t2 127.0.0.1 3306 table test.t2 not exists for 1-th check, retry later table test.t2 exists + sleep 10 [2025/04/04 20:24:17.103 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 492, b: 492, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:54, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: B, d: 20:31:54, a: 492, b: 492, }"] [2025/04/04 20:24:17.204 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 493, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:55, a: 493, }"] [row2="{ a: 493, b: 493, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: W, d: 20:31:55, }"] [2025/04/04 20:24:17.304 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:31:56, a: 494, b: 494, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: WSIx, d: 20:31:56, a: 494, b: 494, }"] [2025/04/04 20:24:17.404 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 495, b: 495, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:57, }"] [row2="{ c: PxDMOMf7a, d: 20:31:57, a: 495, b: 495, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:17.504 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:31:59, a: 497, b: 497, }"] [row2="{ d: 20:31:59, a: 497, b: 497, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: NCAB3Hdft, }"] wait process 8605 exit for 1-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 [2025/04/04 20:24:17.604 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 498, b: 498, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:00, }"] [row2="{ a: 498, b: 498, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: yv9, d: 20:32:00, }"] [2025/04/04 20:24:17.704 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:32:01, a: 499, b: 499, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 499, b: 499, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 7PHRMPhOu, d: 20:32:01, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:17.805 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:32:05, a: 500, b: 500, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:32:05, a: 500, b: 500, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Rriob0UW4, }"] [2025/04/04 20:24:17.905 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 501, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:06, a: 501, }"] [row2="{ b: 501, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Y, d: 20:32:06, a: 501, }"] [2025/04/04 20:24:18.005 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 502, b: 502, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:07, }"] [row2="{ a: 502, b: 502, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: ND4yX, d: 20:32:07, }"] check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process 8605 exit for 2-th time... [2025/04/04 20:24:18.105 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:08, a: 503, b: 503, }"] [row2="{ b: 503, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 0, d: 20:32:08, a: 503, }"] [2025/04/04 20:24:18.205 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:09, a: 504, b: 504, }"] [row2="{ a: 504, b: 504, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 0SrihP, d: 20:32:09, }"] [2025/04/04 20:24:18.305 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 505, b: 505, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:10, }"] [row2="{ a: 505, b: 505, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: W, d: 20:32:10, }"] [2025/04/04 20:24:18.405 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 506, b: 506, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:23, }"] [row2="{ a: 506, b: 506, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: wpO, d: 20:38:23, }"] [2025/04/04 20:24:18.506 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:12, a: 507, b: 507, }"] [row2="{ a: 507, b: 507, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: FGLvfuy9XZ, d: 20:32:12, }"] table consistent_replicate_storage_file.t2 exists 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:18.428318+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:18.460806+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:18.519725+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8605) - No such process wait process 8605 exit for 3-th time... process 8605 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 20:24:18 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.87348736.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:18.606 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 508, b: 508, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:13, }"] [row2="{ a: 508, b: 508, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 6u4x7YCGix, d: 20:32:13, }"] [2025/04/04 20:24:18.706 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:15, a: 510, b: 510, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later check diff successfully [2025/04/04 20:24:18.806 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 511, b: 511, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:16, }"] [2025/04/04 20:24:18.906 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 512, b: 512, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:17, }"] [2025/04/04 20:24:19.006 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:18, a: 513, b: 513, }"] [2025/04/04 20:24:19.107 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:19, a: 514, b: 514, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:19.207 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 515, b: 515, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:20, }"] wait process cdc.test exit for 1-th time... [Fri Apr 4 20:24:18 CST 2025] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3149331495.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... [2025/04/04 20:24:19.307 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:21, a: 516, b: 516, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:19.407 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:22, a: 517, b: 517, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:19.507 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 518, b: 518, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:23, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432770177295585 TiDB Cluster ID. tikv_gc_leader_uuid 658030952240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:35331, start at 2025-04-04 20:24:17.833202971 +0800 CST m=+2.326014600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:17.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:17.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:17.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432770177295585 TiDB Cluster ID. tikv_gc_leader_uuid 658030952240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:35331, start at 2025-04-04 20:24:17.833202971 +0800 CST m=+2.326014600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:17.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:17.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:17.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432771668135318 TiDB Cluster ID. tikv_gc_leader_uuid 658030954a8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:35410, start at 2025-04-04 20:24:17.985117935 +0800 CST m=+2.417741274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:17.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:17.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:17.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/04 20:24:19.607 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 519, b: 519, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:24, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:19.708 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 520, b: 520, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:25, }"] [row2="{ a: 520, b: 520, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: AKuy9l, d: 20:32:25, }"] changefeed is working normally rts: 457118700803719200->457118701066125300 checkpoint: 457118700803719200->457118701066125300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/04 20:24:16.329 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [duration=30.686993ms] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] run task successfully wait process cdc.test exit for 2-th time... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:24:19.808 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:32:26, a: 521, b: 521, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:32:26, a: 521, b: 521, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: T7nca0V, }"] [2025/04/04 20:24:19.908 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 522, b: 522, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:27, }"] [row2="{ d: 20:32:27, a: 522, b: 522, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: L8U, }"] [2025/04/04 20:24:20.008 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:32:28, a: 523, b: 523, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:32:28, a: 523, b: 523, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: AfbX, }"] [2025/04/04 20:24:20.108 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 524, b: 524, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:29, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: MO, d: 20:32:29, a: 524, b: 524, }"] [2025/04/04 20:24:20.208 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 525, b: 525, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:30, }"] [row2="{ c: cQO4169Zh, d: 20:32:30, a: 525, b: 525, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release resume changefeed 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 from 457118699794464787 check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 wait process cdc.test exit for 3-th time... [2025/04/04 20:24:20.309 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:32:31, a: 526, b: 526, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:32:31, a: 526, b: 526, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: DtWNe, }"] [2025/04/04 20:24:20.409 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:32:32, a: 527, b: 527, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 527, b: 527, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 1rqW, d: 20:32:32, }"] [2025/04/04 20:24:20.509 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 528, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:33, a: 528, }"] [row2="{ a: 528, b: 528, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: IE, d: 20:32:33, }"] [2025/04/04 20:24:20.609 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:32:34, a: 529, b: 529, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: PBg4V, d: 20:32:34, a: 529, b: 529, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:20.709 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:35, a: 530, b: 530, }"] [2025/04/04 20:24:20.809 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:32:36, a: 531, b: 531, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:24:20.909 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:32:37, a: 532, b: 532, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:24:21.010 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 533, b: 533, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:38, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:24:20 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1162911632.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:21.110 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:39, a: 534, b: 534, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:21.210 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 535, b: 535, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:40, }"] [2025/04/04 20:24:21.310 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:32:41, a: 536, b: 536, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 [2025/04/04 20:24:21.410 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 537, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:42, a: 537, }"] [2025/04/04 20:24:21.510 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 538, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:43, a: 538, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:21.611 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 539, b: 539, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:44, }"] [2025/04/04 20:24:21.711 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:32:45, a: 540, b: 540, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 540, QH1PS: YMRWsDfmsCfyNObu6R5XEUx06Y8HqoDaovriv2hNOx, c: 2Wv5F, d: 20:32:45, a: 540, }"] [2025/04/04 20:24:21.811 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 541, b: 541, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:46, }"] [row2="{ b: 541, QH1PS: 4gIsIrGIwMTCbeD, c: KlaQqTy, d: 20:32:46, a: 541, }"] The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting TLS PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e48090 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/229d58ea-8536-4431-b4ea-7624a965ba84 {"id":"229d58ea-8536-4431-b4ea-7624a965ba84","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769459} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ecf1 229d58ea-8536-4431-b4ea-7624a965ba84 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/229d58ea-8536-4431-b4ea-7624a965ba84/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e48090 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/229d58ea-8536-4431-b4ea-7624a965ba84 {"id":"229d58ea-8536-4431-b4ea-7624a965ba84","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769459} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ecf1 229d58ea-8536-4431-b4ea-7624a965ba84 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/229d58ea-8536-4431-b4ea-7624a965ba84/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 + grep -q 'failed to get info:' {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e48090 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692486414349 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/229d58ea-8536-4431-b4ea-7624a965ba84 {"id":"229d58ea-8536-4431-b4ea-7624a965ba84","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769459} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ecf1 229d58ea-8536-4431-b4ea-7624a965ba84 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692486414349,"admin-job-type":0} /tidb/cdc/default/default/task/position/229d58ea-8536-4431-b4ea-7624a965ba84/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2025/04/04 20:24:21.911 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 542, b: 542, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:47, }"] [row2="{ d: 20:32:47, a: 542, b: 542, QH1PS: X3LFBsuByvMuijz0oR0EZPtEmJ4gNDW, c: Kwh87uhUuH, }"] [2025/04/04 20:24:22.012 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:32:48, a: 543, b: 543, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 543, b: 543, QH1PS: Z3f26phLUssy2bThOPvUqVdhNOHy, c: 9, d: 20:32:48, }"] Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:21 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4 {"id":"74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769459} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c22b9ed4 74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4 /tidb/cdc/default/default/upstream/7489432759072875547 {"id":7489432759072875547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4 {"id":"74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769459} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c22b9ed4 74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4 /tidb/cdc/default/default/upstream/7489432759072875547 {"id":7489432759072875547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4 {"id":"74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769459} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c22b9ed4 74f5c50f-6f8f-4de5-983f-8c8d3ad73fb4 /tidb/cdc/default/default/upstream/7489432759072875547 {"id":7489432759072875547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 30cec0a9-9ad4-4931-83d0-85847be5bbb4 Info: {"upstream_id":7489432759072875547,"namespace":"default","id":"30cec0a9-9ad4-4931-83d0-85847be5bbb4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:24:22.089147427+08:00","start_ts":457118701855178754,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118701855178754,"checkpoint_ts":457118701855178754,"checkpoint_time":"2025-04-04 20:24:22.033"} Starting Upstream TiDB... [2025/04/04 20:24:22.112 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:32:49, a: 544, b: 544, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 544, b: 544, QH1PS: X, c: wmgWguVB, d: 20:32:49, }"] [2025/04/04 20:24:22.212 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 545, b: 545, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:50, }"] [row2="{ a: 545, b: 545, QH1PS: JP11CjuVBqVDm3rEYSJRrzlu, c: Bef2Z5K, d: 20:32:50, }"] [2025/04/04 20:24:22.312 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:32:51, a: 546, b: 546, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 546, b: 546, QH1PS: br3sNvKrdyiK8NT3r, c: OhV9, d: 20:32:51, }"] \033[0;36m<<< Run all test success >>>\033[0m run task failed 1-th time, retry later table autorandom_test.table_a not exists for 1-th check, retry later [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:22.412 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:32:54, a: 549, b: 549, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ QH1PS: eadjYWdsLzG6e, c: xBWw, d: 20:32:54, a: 549, b: 549, }"] [2025/04/04 20:24:22.512 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:32:55, a: 550, b: 550, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 [2025/04/04 20:24:22.613 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 551, b: 551, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:56, }"] [2025/04/04 20:24:22.713 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 552, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:57, a: 552, }"] [2025/04/04 20:24:22.813 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:58, a: 553, b: 553, }"] [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2025/04/04 20:24:22.913 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 554, b: 554, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:32:59, }"] [2025/04/04 20:24:23.013 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:33:00, a: 555, b: 555, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [2025/04/04 20:24:23.114 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 556, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:01, a: 556, }"] [2025/04/04 20:24:23.214 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 557, b: 557, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:02, }"] [2025/04/04 20:24:23.314 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 558, b: 558, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:03, }"] [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + run_sql 'SELECT * from test.t2' 127.0.0.1 3306 + check_contains 'a: 2023-02-08' + stop_tidb_cluster [2025/04/04 20:24:23.414 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:33:04, a: 559, b: 559, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:24:23.514 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:33:05, a: 560, b: 560, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 560, b: 560, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: LL, d: 20:33:05, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 [2025/04/04 20:24:23.614 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 561, b: 561, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:06, }"] [row2="{ a: 561, b: 561, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: qOX, d: 20:33:06, }"] [2025/04/04 20:24:23.714 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 562, b: 562, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:54, }"] [row2="{ c: Aju3, d: 20:38:54, a: 562, b: 562, QH1PS: 7OFUHTHvaSwQIOy9o8, }"] [2025/04/04 20:24:23.815 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:08, a: 563, b: 563, }"] [row2="{ a: 563, b: 563, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: n8V0uFJOs, d: 20:33:08, }"] [2025/04/04 20:24:23.915 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 564, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:09, a: 564, }"] [row2="{ c: z96Ke, d: 20:33:09, a: 564, b: 564, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:24.015 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 565, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:10, a: 565, }"] [row2="{ d: 20:33:10, a: 565, b: 565, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: XXse1, }"] check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 table autorandom_test.table_a exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/173e4daf-f096-499f-b960-35825eb11c1a {UpstreamID:7489432515505163827 Namespace:default ID:173e4daf-f096-499f-b960-35825eb11c1a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:27.503339613 +0800 CST StartTs:457118687544737795 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d23b90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118687557582850} {CheckpointTs:457118688108347433 MinTableBarrierTs:457118702145896456 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a7fdc36-3086-4da9-9857-8fa3b56a5443 {"id":"7a7fdc36-3086-4da9-9857-8fa3b56a5443","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769461} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1546f0a 7a7fdc36-3086-4da9-9857-8fa3b56a5443 /tidb/cdc/default/default/changefeed/info/173e4daf-f096-499f-b960-35825eb11c1a {"upstream-id":7489432515505163827,"namespace":"default","changefeed-id":"173e4daf-f096-499f-b960-35825eb11c1a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:23:27.503339613+08:00","start-ts":457118687544737795,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118687557582850} /tidb/cdc/default/default/changefeed/status/173e4daf-f096-499f-b960-35825eb11c1a {"checkpoint-ts":457118688108347433,"min-table-barrier-ts":457118702145896456,"admin-job-type":0} /tidb/cdc/default/default/task/position/7a7fdc36-3086-4da9-9857-8fa3b56a5443/173e4daf-f096-499f-b960-35825eb11c1a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432515505163827 {"id":7489432515505163827,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/173e4daf-f096-499f-b960-35825eb11c1a {UpstreamID:7489432515505163827 Namespace:default ID:173e4daf-f096-499f-b960-35825eb11c1a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:27.503339613 +0800 CST StartTs:457118687544737795 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d23b90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118687557582850} {CheckpointTs:457118688108347433 MinTableBarrierTs:457118702145896456 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a7fdc36-3086-4da9-9857-8fa3b56a5443 {"id":"7a7fdc36-3086-4da9-9857-8fa3b56a5443","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769461} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1546f0a 7a7fdc36-3086-4da9-9857-8fa3b56a5443 /tidb/cdc/default/default/changefeed/info/173e4daf-f096-499f-b960-35825eb11c1a {"upstream-id":7489432515505163827,"namespace":"default","changefeed-id":"173e4daf-f096-499f-b960-35825eb11c1a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:23:27.503339613+08:00","start-ts":457118687544737795,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118687557582850} /tidb/cdc/default/default/changefeed/status/173e4daf-f096-499f-b960-35825eb11c1a {"checkpoint-ts":457118688108347433,"min-table-barrier-ts":457118702145896456,"admin-job-type":0} /tidb/cdc/default/default/task/position/7a7fdc36-3086-4da9-9857-8fa3b56a5443/173e4daf-f096-499f-b960-35825eb11c1a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432515505163827 {"id":7489432515505163827,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/173e4daf-f096-499f-b960-35825eb11c1a {UpstreamID:7489432515505163827 Namespace:default ID:173e4daf-f096-499f-b960-35825eb11c1a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:23:27.503339613 +0800 CST StartTs:457118687544737795 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d23b90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118687557582850} {CheckpointTs:457118688108347433 MinTableBarrierTs:457118702145896456 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118688108347433, checkpointTs: 457118688108347433, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a7fdc36-3086-4da9-9857-8fa3b56a5443 {"id":"7a7fdc36-3086-4da9-9857-8fa3b56a5443","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769461} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c1546f0a 7a7fdc36-3086-4da9-9857-8fa3b56a5443 /tidb/cdc/default/default/changefeed/info/173e4daf-f096-499f-b960-35825eb11c1a {"upstream-id":7489432515505163827,"namespace":"default","changefeed-id":"173e4daf-f096-499f-b960-35825eb11c1a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:23:27.503339613+08:00","start-ts":457118687544737795,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118687557582850} /tidb/cdc/default/default/changefeed/status/173e4daf-f096-499f-b960-35825eb11c1a {"checkpoint-ts":457118688108347433,"min-table-barrier-ts":457118702145896456,"admin-job-type":0} /tidb/cdc/default/default/task/position/7a7fdc36-3086-4da9-9857-8fa3b56a5443/173e4daf-f096-499f-b960-35825eb11c1a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432515505163827 {"id":7489432515505163827,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [2025/04/04 20:24:24.116 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 566, b: 566, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:11, }"] [row2="{ d: 20:33:11, a: 566, b: 566, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: q, }"] [2025/04/04 20:24:24.216 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 567, b: 567, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:12, }"] [row2="{ a: 567, b: 567, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: Q, d: 20:33:12, }"] [2025/04/04 20:24:24.316 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:13, a: 568, b: 568, }"] [row2="{ a: 568, b: 568, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: WlBfopBi6, d: 20:33:13, }"] check diff failed 1-th time, retry later [2025/04/04 20:24:24.416 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:33:14, a: 569, b: 569, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: rG9LJ, d: 20:33:14, a: 569, b: 569, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:24.516 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 570, b: 570, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:15, }"] [row2="{ c: pcJMIQJrHt, d: 20:33:15, a: 570, b: 570, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:24.429858+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:24.462093+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:24.521373+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:24:24.617 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:16, a: 571, b: 571, }"] [row2="{ c: N, d: 20:33:16, a: 571, b: 571, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] [2025/04/04 20:24:24.717 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 572, b: 572, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:17, }"] [row2="{ a: 572, b: 572, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: u, d: 20:33:17, }"] [2025/04/04 20:24:24.817 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 573, b: 573, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:18, }"] [row2="{ c: m, d: 20:33:18, a: 573, b: 573, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:24.917 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 574, b: 574, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:19, }"] [row2="{ a: 574, b: 574, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 3x96GUxm, d: 20:33:19, }"] [2025/04/04 20:24:25.017 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:33:20, a: 575, b: 575, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 575, b: 575, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: oqq, d: 20:33:20, }"] [2025/04/04 20:24:25.117 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 576, b: 576, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:21, }"] [row2="{ a: 576, b: 576, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: cpWgb9I4PM, d: 20:33:21, }"] nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully wait process cdc.test exit for 1-th time... [2025/04/04 20:24:25.218 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 577, b: 577, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:22, }"] [row2="{ QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: 2wUHiBNd, d: 20:33:22, a: 577, b: 577, }"] [2025/04/04 20:24:25.318 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 578, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:23, a: 578, }"] [row2="{ a: 578, b: 578, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: p, d: 20:33:23, }"] Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:24:25.418 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 579, b: 579, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:33:24, }"] [row2="{ a: 579, b: 579, QH1PS: strqXR40PGeDps8fPIVVu9YSeFMmHrGORZNPZZTpc4i7W, c: sJ, d: 20:33:24, }"] [2025/04/04 20:24:25.518 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 580, b: 580, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: vxCny, d: 20:33:25, }"] [2025/04/04 20:24:25.618 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 581, b: 581, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: TZaENM, d: 20:33:26, }"] [2025/04/04 20:24:25.719 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:33:27, a: 582, b: 582, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: qBqr, }"] [2025/04/04 20:24:25.819 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 583, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: bTIGwsSyBe, d: 20:33:28, a: 583, }"] wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:25.919 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: LXr, d: 20:33:29, a: 584, b: 584, }"] [2025/04/04 20:24:26.019 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 585, b: 585, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: 6AoK, d: 20:33:30, }"] [2025/04/04 20:24:26.119 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: 4P, d: 20:33:31, a: 586, b: 586, }"] changefeed is working normally rts: 457118702376583200->457118702638989300 checkpoint: 457118702376583200->457118702638989300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2025/04/04 20:24:21.160 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457118699794464784] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [error="Error 1050 (42S01): Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4 without validation, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` WITHOUT VALIDATION check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:24:26.219 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 587, b: 587, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: IFWX, d: 20:33:32, }"] [2025/04/04 20:24:26.320 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 588, b: 588, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: OaCESUO, d: 20:33:33, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_capture [Fri Apr 4 20:24:26 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.60866088.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:26.41771+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T20:24:26.417757+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T20:24:26.417801+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T20:24:26.417844+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:26.507061+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-04T20:24:26.507143+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [2025/04/04 20:24:26.420 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 589, b: 589, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: lSI0t, d: 20:33:34, }"] [2025/04/04 20:24:26.520 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:33:59, a: 610, b: 610, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: su21D3GB, }"] [2025/04/04 20:24:26.620 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 611, b: 611, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: wTxQIvSh, d: 20:34:00, }"] Starting TLS TiDB... [2025/04/04 20:24:26.720 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: 8Kv, d: 20:34:01, a: 612, b: 612, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:26.820 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: maDLDIjp, d: 20:34:02, a: 613, b: 613, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432797585339557 TiDB Cluster ID. tikv_gc_leader_uuid 6580309c0b40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:15588, start at 2025-04-04 20:24:24.885448396 +0800 CST m=+2.515959544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:24.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:24.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:24.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432797585339557 TiDB Cluster ID. tikv_gc_leader_uuid 6580309c0b40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:15588, start at 2025-04-04 20:24:24.885448396 +0800 CST m=+2.515959544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:24.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:24.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:24.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432798742324879 TiDB Cluster ID. tikv_gc_leader_uuid 6580309c0a80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:15654, start at 2025-04-04 20:24:24.887082664 +0800 CST m=+2.431631174 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:24.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:24.925 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:24.925 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... [2025/04/04 20:24:26.921 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:34:03, a: 614, b: 614, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: yc0mC, }"] [2025/04/04 20:24:27.021 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 615, b: 615, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: 6CRk7BWWUz, d: 20:34:04, }"] [2025/04/04 20:24:27.121 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 616, b: 616, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: LAd8v, d: 20:34:05, }"] [2025/04/04 20:24:27.221 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: F2KSU, d: 20:34:06, a: 617, b: 617, }"] [2025/04/04 20:24:27.321 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 618, b: 618, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: LUq8wOmf, d: 20:34:07, }"] wait process cdc.test exit for 2-th time... [2025/04/04 20:24:27.422 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 619, b: 619, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: f, d: 20:34:08, }"] [2025/04/04 20:24:27.522 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 620, QH1PS: 7DLmxObmbHO8Aa7YL4ViQCEKTPLqjr, c: j0vl, d: 20:34:09, a: 620, }"] [2025/04/04 20:24:27.622 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: Vt, d: 20:34:10, a: 621, b: 621, QH1PS: jF, }"] [2025/04/04 20:24:27.722 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 622, b: 622, QH1PS: f91PWwwWyrV0FZfz8V5P3oMpzp, c: H0gN36UH9t, d: 20:34:11, }"] [2025/04/04 20:24:27.822 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 623, b: 623, QH1PS: VuWWibeKKr9s6lB0K57yceya, c: Hw0ytk9G, d: 20:34:12, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed is working normally rts: 457118702900871200->457118703163015200 checkpoint: 457118702900871200->457118703163015200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/04 20:24:27.405 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [duration=50.466441ms] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` WITHOUT VALIDATION"] run task successfully [2025/04/04 20:24:27.922 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 624, b: 624, QH1PS: uSlRPVyfNhLAIqfxcc8, c: fTiaxas, d: 20:34:13, }"] [2025/04/04 20:24:28.023 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 625, b: 625, QH1PS: 0rEf4E0WE0TQPw, c: VSctbgW, d: 20:34:14, }"] [2025/04/04 20:24:28.123 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 626, b: 626, QH1PS: 0H5En51yZMsLSjnzzj39EPqpMRIx0, c: B, d: 20:34:15, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:24:28 CST 2025] <<<<<< run test case autorandom success! >>>>>> [2025/04/04 20:24:28.223 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 627, b: 627, QH1PS: oEKD8uI, c: yVsp, d: 20:34:16, }"] [2025/04/04 20:24:28.323 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 628, b: 628, QH1PS: Us77yl7L5kX3p2ShF7pS6c6RffQYfTy, c: O2OqF, d: 20:34:17, }"] resume changefeed 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 from 457118702900871205 check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 [2025/04/04 20:24:28.423 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: WkyNAu9Fy8W80r, c: H9, d: 20:34:18, a: 629, b: 629, }"] [2025/04/04 20:24:28.523 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:34:29, a: 640, b: 640, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:34:29, a: 640, b: 640, QH1PS: 80aT2yObZogzLibDylgFYiPV9YtvDFpf7onxg, c: Fh3, }"] [2025/04/04 20:24:28.624 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 641, b: 641, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:30, }"] [row2="{ c: mUHq2tsON, d: 20:34:30, a: 641, b: 641, QH1PS: o0XLUwhlqNyIP0Knhu6QJzv3KmBFy, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16896.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432815991351305 TiDB Cluster ID. tikv_gc_leader_uuid 6580309f55c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:36946, start at 2025-04-04 20:24:28.28788481 +0800 CST m=+1.484456378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:28.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:28.296 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:28.296 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/04 20:24:28.724 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 642, b: 642, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:31, }"] [row2="{ QH1PS: UnfSgGrGShBdBms2YLGogeBTqcAP, c: IEbw, d: 20:34:31, a: 642, b: 642, }"] [2025/04/04 20:24:28.824 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:34:32, a: 643, b: 643, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 643, QH1PS: uwfG3RPSMnGyr38bw, c: HMDstRg0nY, d: 20:34:32, a: 643, }"] [2025/04/04 20:24:28.924 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:33, a: 644, b: 644, }"] [row2="{ d: 20:34:33, a: 644, b: 644, QH1PS: dFjYWsdoBdsiMNeQeHWjgm3akj9tY5x6rAJCepOjlQTUD, c: D1, }"] [Fri Apr 4 20:24:28 CST 2025] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.3700137003.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:29.024 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 645, b: 645, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:34, }"] [row2="{ a: 645, b: 645, QH1PS: 2pPqZ8up54PT0KB1oa, c: fXWE4JDd, d: 20:34:34, }"] [2025/04/04 20:24:29.125 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:34:35, a: 646, b: 646, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 646, b: 646, QH1PS: 9U59JcNsiGd5Ks8S0eE9t0bZVwUGH8MLAc6, c: s08SA, d: 20:34:35, }"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:24:29.225 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:34:36, a: 647, b: 647, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:34:36, a: 647, b: 647, QH1PS: P2Prx7j2qtG2HiTbopLGsZIX8YXFEowWMsfkUU, c: 4ET3t, }"] [2025/04/04 20:24:29.325 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 648, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:37, a: 648, }"] [row2="{ a: 648, b: 648, QH1PS: Xo6BqUIc5O0VQfu1, c: 9ydekXG, d: 20:34:37, }"] [2025/04/04 20:24:29.425 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:38, a: 649, b: 649, }"] [row2="{ a: 649, b: 649, QH1PS: g9pwN873I5zIljBM1Sej, c: VDG8AomBG2, d: 20:34:38, }"] [2025/04/04 20:24:29.525 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 650, b: 650, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:39, }"] [row2="{ a: 650, b: 650, QH1PS: rAm8OEo, c: W5vzcj, d: 20:34:39, }"] [2025/04/04 20:24:29.625 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:40, a: 651, b: 651, }"] [row2="{ a: 651, b: 651, QH1PS: T1FMrEAkMqQIigPIuY8GwrMxnYpYuvn6IJNJyE, c: B, d: 20:34:40, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003535dd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118703436431364 MinTableBarrierTs:457118703698313221 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118703698313221, checkpointTs: 457118703698313221, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118703698313220, checkpointTs: 457118703698313220, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118703698313223, checkpointTs: 457118703698313221, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22726268-44ad-42ad-83da-cef520863f37 {"id":"22726268-44ad-42ad-83da-cef520863f37","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769466} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89f7d 22726268-44ad-42ad-83da-cef520863f37 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118703436431364,"min-table-barrier-ts":457118703698313221,"admin-job-type":0} /tidb/cdc/default/default/task/position/22726268-44ad-42ad-83da-cef520863f37/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003535dd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118703436431364 MinTableBarrierTs:457118703698313221 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118703698313221, checkpointTs: 457118703698313221, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118703698313220, checkpointTs: 457118703698313220, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118703698313223, checkpointTs: 457118703698313221, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22726268-44ad-42ad-83da-cef520863f37 {"id":"22726268-44ad-42ad-83da-cef520863f37","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769466} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89f7d 22726268-44ad-42ad-83da-cef520863f37 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118703436431364,"min-table-barrier-ts":457118703698313221,"admin-job-type":0} /tidb/cdc/default/default/task/position/22726268-44ad-42ad-83da-cef520863f37/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003535dd0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118703436431364 MinTableBarrierTs:457118703698313221 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118703698313221, checkpointTs: 457118703698313221, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118703698313220, checkpointTs: 457118703698313220, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118703698313223, checkpointTs: 457118703698313221, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22726268-44ad-42ad-83da-cef520863f37 {"id":"22726268-44ad-42ad-83da-cef520863f37","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769466} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89f7d 22726268-44ad-42ad-83da-cef520863f37 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118703436431364,"min-table-barrier-ts":457118703698313221,"admin-job-type":0} /tidb/cdc/default/default/task/position/22726268-44ad-42ad-83da-cef520863f37/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:29.726 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 653, b: 653, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:42, }"] [row2="{ QH1PS: w52Bx1O, c: iDdfM1ZQF, d: 20:34:42, a: 653, b: 653, }"] [2025/04/04 20:24:29.826 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 654, b: 654, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:43, }"] [row2="{ a: 654, b: 654, QH1PS: 8XfJFde46OszzAOca0, c: FliQdf, d: 20:34:43, }"] [2025/04/04 20:24:29.926 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 655, b: 655, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:44, }"] [row2="{ a: 655, b: 655, QH1PS: PAA2qtjW3XPs3QfKBzF7XN2hV39MbSA0CVPs5wbYx, c: kiZHQQ9V4j, d: 20:34:44, }"] [2025/04/04 20:24:30.026 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 656, b: 656, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:45, }"] [row2="{ c: OgwZ, d: 20:34:45, a: 656, b: 656, QH1PS: 70NhnXpIUKjLw, }"] [2025/04/04 20:24:30.126 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 657, b: 657, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:46, }"] [row2="{ a: 657, b: 657, QH1PS: ieHirXE, c: Q73cnug1, d: 20:34:46, }"] [2025/04/04 20:24:30.226 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:34:47, a: 658, b: 658, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 658, b: 658, QH1PS: t4vzrIrpbts5fTJSsHNCPVZaAbHOpCig9epEQa, c: ovZK9, d: 20:34:47, }"] [2025/04/04 20:24:30.327 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 659, b: 659, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:48, }"] [row2="{ a: 659, b: 659, QH1PS: CpnKa6keXV9LuqlF, c: lvsl, d: 20:34:48, }"] [2025/04/04 20:24:30.427 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:50, a: 660, b: 660, }"] [row2="{ QH1PS: 20smSfA9pO6jrtkApEDdDDlZjWyqnJPrU, c: m5, d: 20:34:50, a: 660, b: 660, }"] run task failed 1-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:30.430302+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:30.46384+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:30.522828+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:24:30.527 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 661, b: 661, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:51, }"] [row2="{ d: 20:34:51, a: 661, b: 661, QH1PS: O, c: JaAj1uCGwI, }"] [2025/04/04 20:24:30.627 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 662, b: 662, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:52, }"] [row2="{ QH1PS: uByHtAYWAR8uIgu4xl8jjQ2ku, c: wj9, d: 20:34:52, a: 662, b: 662, }"] + set +x + tso='457118703701721091 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118703701721091 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118703701721091 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 20:24:30 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1693716939.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:30.727 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 663, b: 663, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:53, }"] [row2="{ b: 663, QH1PS: OyOc2itsGy3Ii2ORoYEIWyJoC2UOkhZrwaikm5, c: 9YrGA, d: 20:34:53, a: 663, }"] [2025/04/04 20:24:30.827 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:34:54, a: 664, b: 664, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 664, QH1PS: ImRF77, c: Oufbe5nR, d: 20:34:54, a: 664, }"] [2025/04/04 20:24:30.928 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 665, b: 665, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:55, }"] [row2="{ a: 665, b: 665, QH1PS: 542EN, c: pdGj, d: 20:34:55, }"] "is-owner": true, run task successfully [2025/04/04 20:24:31.028 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:34:56, a: 666, b: 666, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: w, d: 20:34:56, a: 666, b: 666, QH1PS: VRHeDIBhwY30caqxhJEcssfVH5ASTAE2QpZyd9d, }"] [2025/04/04 20:24:31.128 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 667, b: 667, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:57, }"] [row2="{ a: 667, b: 667, QH1PS: vL33F5NoO2bwGp2, c: 8eX3h, d: 20:34:57, }"] [2025/04/04 20:24:31.228 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:58, a: 668, b: 668, }"] [row2="{ a: 668, b: 668, QH1PS: 2sfjzawFHnDzDjVwImqpStV67OroH, c: rdoHWc3a, d: 20:34:58, }"] [2025/04/04 20:24:31.328 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 669, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:34:59, a: 669, }"] [row2="{ a: 669, b: 669, QH1PS: Jgue, c: 4ZCu2N, d: 20:34:59, }"] [2025/04/04 20:24:31.428 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:35:00, a: 670, b: 670, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 670, b: 670, QH1PS: kqnbLLPgAUZ3lMZhlnzxnr, c: TkFQs97, d: 20:35:00, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:31.528 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:35:01, a: 671, b: 671, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 671, QH1PS: HbVeKWztvyxbGVmlIOlvHdiTSXrtlD1iGIU035YZv1, c: Xo, d: 20:35:01, a: 671, }"] [2025/04/04 20:24:31.629 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 672, b: 672, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:02, }"] [row2="{ a: 672, b: 672, QH1PS: ZoNJwaoplGT, c: u945JVFOq, d: 20:35:02, }"] [2025/04/04 20:24:31.729 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 673, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:03, a: 673, }"] [row2="{ d: 20:35:03, a: 673, b: 673, QH1PS: ZbhxO6z4B4Uzy7mhuZ6Ee2wVAZIPXKS3Aine5imBk1Qy, c: tKBs9, }"] [2025/04/04 20:24:31.829 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 674, b: 674, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:04, }"] [row2="{ a: 674, b: 674, QH1PS: 9TL, c: nSItU, d: 20:35:04, }"] [2025/04/04 20:24:31.929 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 675, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:05, a: 675, }"] [row2="{ a: 675, b: 675, QH1PS: Z1dH7EfmMMi1vX8EFGpj, c: DZlcGs4, d: 20:35:05, }"] /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8739) - No such process wait process 8739 exit for 1-th time... process 8739 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 20:24:30 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.88718873.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:32.029 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 676, b: 676, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:06, }"] [row2="{ a: 676, b: 676, QH1PS: FutXjkgSg3dAuBeRsB3o70Kqel7th, c: wc, d: 20:35:06, }"] [2025/04/04 20:24:32.129 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:39:36, a: 677, b: 677, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 677, b: 677, QH1PS: ykqyDqZKRbyM2aqXczGVOYpKlsRh, c: mN6PKrBF, d: 20:35:07, }"] check_ts_forward 894a28ee-3ad5-4e3c-8b31-6dddbab8eab9 + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:32 GMT < Content-Length: 1253 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0569f733-cd91-4619-a474-5a4b06d0cb66 {"id":"0569f733-cd91-4619-a474-5a4b06d0cb66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264dfdd 0569f733-cd91-4619-a474-5a4b06d0cb66 /tidb/cdc/default/default/upstream/7489432815991351305 {"id":7489432815991351305,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0569f733-cd91-4619-a474-5a4b06d0cb66 {"id":"0569f733-cd91-4619-a474-5a4b06d0cb66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264dfdd 0569f733-cd91-4619-a474-5a4b06d0cb66 /tidb/cdc/default/default/upstream/7489432815991351305 {"id":7489432815991351305,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0569f733-cd91-4619-a474-5a4b06d0cb66 {"id":"0569f733-cd91-4619-a474-5a4b06d0cb66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264dfdd 0569f733-cd91-4619-a474-5a4b06d0cb66 /tidb/cdc/default/default/upstream/7489432815991351305 {"id":7489432815991351305,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x owner pid: 6091 owner id 22726268-44ad-42ad-83da-cef520863f37 [Fri Apr 4 20:24:32 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.62016203.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:32.230 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 678, b: 678, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:08, }"] [row2="{ c: V, d: 20:35:08, a: 678, b: 678, QH1PS: pXr3B, }"] [2025/04/04 20:24:32.330 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 679, b: 679, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:09, }"] [row2="{ QH1PS: Qlh4wUaVw, c: TY, d: 20:35:09, a: 679, b: 679, }"] [2025/04/04 20:24:32.430 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 680, b: 680, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:10, }"] [row2="{ b: 680, QH1PS: nxUkywqDrhinmoWJUcp5CQxf, c: WxQ5evsmLO, d: 20:35:10, a: 680, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 [2025/04/04 20:24:32.530 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 681, b: 681, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:11, }"] [row2="{ d: 20:35:11, a: 681, b: 681, QH1PS: UWbPZNiuM, c: 6PDf4F, }"] [2025/04/04 20:24:32.630 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 682, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:12, a: 682, }"] [row2="{ a: 682, b: 682, QH1PS: 1LRHTMqJ7zWEq9I, c: nNMe5Sxs, d: 20:35:12, }"] [2025/04/04 20:24:32.730 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 683, b: 683, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:13, }"] [row2="{ a: 683, b: 683, QH1PS: gs7, c: pEc4Fgh7t, d: 20:35:13, }"] [2025/04/04 20:24:32.830 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 684, b: 684, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:14, }"] [row2="{ c: AQ, d: 20:35:14, a: 684, b: 684, QH1PS: SA092Fx, }"] [2025/04/04 20:24:32.931 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 685, b: 685, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:15, }"] [row2="{ a: 685, b: 685, QH1PS: Pm14Yd1BqHlWhV6Zz, c: zI1MhdwXLv, d: 20:35:15, }"] [2025/04/04 20:24:33.031 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 686, b: 686, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:16, }"] [row2="{ QH1PS: WEwZrfh9f7QxB9jIujuTD8WJdvGGlh7JVoJbTD, c: Ji4, d: 20:35:16, a: 686, b: 686, }"] [2025/04/04 20:24:33.131 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:35:17, a: 687, b: 687, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:35:17, a: 687, b: 687, QH1PS: OFOsMokVWg7NLXfcSQ23HVD, c: W8L8C, }"] [2025/04/04 20:24:33.231 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 688, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:18, a: 688, }"] [row2="{ d: 20:35:18, a: 688, b: 688, QH1PS: baqG05Q7BvmkSl, c: h8D, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 [2025/04/04 20:24:33.331 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:35:19, a: 689, b: 689, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:35:19, a: 689, b: 689, QH1PS: WvNnzfrt7sRxGUX, c: NjxN, }"] [2025/04/04 20:24:33.432 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 690, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:20, a: 690, }"] [row2="{ a: 690, b: 690, QH1PS: a56ciwnNrg63pXLIePXE7vrCIRi4S, c: BcG4lPJQ, d: 20:35:20, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:33.532 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 692, b: 692, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:22, }"] [row2="{ a: 692, b: 692, QH1PS: 6GanSu, c: QpV, d: 20:35:22, }"] [2025/04/04 20:24:33.632 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 693, b: 693, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:23, }"] [row2="{ b: 693, QH1PS: gPJlVm, c: h4lfLpiPj, d: 20:35:23, a: 693, }"] [2025/04/04 20:24:33.732 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 694, b: 694, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:24, }"] [row2="{ c: P, d: 20:35:24, a: 694, b: 694, QH1PS: 410waCr4cC9, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042850e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692499259395 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3901ccb-d59e-4e22-87eb-f8932007bf91 {"id":"b3901ccb-d59e-4e22-87eb-f8932007bf91","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ed6c b3901ccb-d59e-4e22-87eb-f8932007bf91 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692499259395,"admin-job-type":0} /tidb/cdc/default/default/task/position/b3901ccb-d59e-4e22-87eb-f8932007bf91/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042850e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692499259395 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3901ccb-d59e-4e22-87eb-f8932007bf91 {"id":"b3901ccb-d59e-4e22-87eb-f8932007bf91","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ed6c b3901ccb-d59e-4e22-87eb-f8932007bf91 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692499259395,"admin-job-type":0} /tidb/cdc/default/default/task/position/b3901ccb-d59e-4e22-87eb-f8932007bf91/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/beed2760-3bba-40f4-bf61-df055e02e635 {UpstreamID:7489432571592914816 Namespace:default ID:beed2760-3bba-40f4-bf61-df055e02e635 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 20:23:39.330649881 +0800 CST StartTs:457118690624929798 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042850e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118690651406343} {CheckpointTs:457118692486414349 MinTableBarrierTs:457118692499259395 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118692486414349, checkpointTs: 457118692486414349, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3901ccb-d59e-4e22-87eb-f8932007bf91 {"id":"b3901ccb-d59e-4e22-87eb-f8932007bf91","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c182ed6c b3901ccb-d59e-4e22-87eb-f8932007bf91 /tidb/cdc/default/default/changefeed/info/beed2760-3bba-40f4-bf61-df055e02e635 {"upstream-id":7489432571592914816,"namespace":"default","changefeed-id":"beed2760-3bba-40f4-bf61-df055e02e635","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T20:23:39.330649881+08:00","start-ts":457118690624929798,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118690651406343} /tidb/cdc/default/default/changefeed/status/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":457118692486414349,"min-table-barrier-ts":457118692499259395,"admin-job-type":0} /tidb/cdc/default/default/task/position/b3901ccb-d59e-4e22-87eb-f8932007bf91/beed2760-3bba-40f4-bf61-df055e02e635 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null+ grep -q 'etcd info' ,"warning":null} /tidb/cdc/default/default/upstream/7489432571592914816 {"id":7489432571592914816,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:33 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfb08c91-9f61-47cb-a1dd-0b267f110e71 {"id":"dfb08c91-9f61-47cb-a1dd-0b267f110e71","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c25439ea dfb08c91-9f61-47cb-a1dd-0b267f110e71 /tidb/cdc/default/default/upstream/7489432797585339557 {"id":7489432797585339557,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfb08c91-9f61-47cb-a1dd-0b267f110e71 {"id":"dfb08c91-9f61-47cb-a1dd-0b267f110e71","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c25439ea dfb08c91-9f61-47cb-a1dd-0b267f110e71 /tidb/cdc/default/default/upstream/7489432797585339557 {"id":7489432797585339557,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dfb08c91-9f61-47cb-a1dd-0b267f110e71 {"id":"dfb08c91-9f61-47cb-a1dd-0b267f110e71","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c25439ea dfb08c91-9f61-47cb-a1dd-0b267f110e71 /tidb/cdc/default/default/upstream/7489432797585339557 {"id":7489432797585339557,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118703701721091 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16996.out cli changefeed create --start-ts=457118703701721091 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml [2025/04/04 20:24:33.833 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:35:25, a: 695, b: 695, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 695, QH1PS: LuArzqcVsKElAvb7OapU7E, c: hMD7bg, d: 20:35:25, a: 695, }"] [2025/04/04 20:24:33.933 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 696, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:26, a: 696, }"] [row2="{ d: 20:35:26, a: 696, b: 696, QH1PS: 7xhRG, c: Eai, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1743769474.022279,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004f70a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769474.0223336,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489432797585339557,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:24:33.984612159+08:00","start_ts":457118703701721091,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118703701721091,"checkpoint_ts":457118703701721091,"checkpoint_time":"2025-04-04 20:24:29.077"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:24:34.034 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 697, b: 697, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:27, }"] [row2="{ c: VUtTn7fBhq, d: 20:35:27, a: 697, b: 697, QH1PS: T4QHVmMuFK, }"] [2025/04/04 20:24:34.134 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 698, b: 698, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:28, }"] [row2="{ a: 698, b: 698, QH1PS: cfrQ, c: q5Gp, d: 20:35:28, }"] [2025/04/04 20:24:34.234 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 699, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:29, a: 699, }"] [row2="{ c: sKVXwdPjh, d: 20:35:29, a: 699, b: 699, QH1PS: HEM45EZ9JISTjzYuqL9Am0k, }"] changefeed is working normally rts: 457118704473997300->457118704735879200 checkpoint: 457118704473997300->457118704735879200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/04 20:24:29.217 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=894a28ee-3ad5-4e3c-8b31-6dddbab8eab9] [duration=47.937471ms] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` WITHOUT VALIDATION"] run task successfully {"level":"warn","ts":1743769474.097118,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0041b8e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743769474.0971878,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769474.111252,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0030a4c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769474.1113048,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [Fri Apr 4 20:24:34 CST 2025] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.3706437066.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:34.334 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:35:30, a: 700, b: 700, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 700, b: 700, QH1PS: AMZnr5, c: p3m, d: 20:35:30, }"] [2025/04/04 20:24:34.434 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 701, b: 701, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:31, }"] [row2="{ b: 701, QH1PS: RIGfKcChsxOnGNouAHPfS1qs9YShRsLsJioEiq, c: d, d: 20:35:31, a: 701, }"] [2025/04/04 20:24:34.534 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 702, b: 702, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:32, }"] [row2="{ a: 702, b: 702, QH1PS: 72VC, c: w, d: 20:35:32, }"] [2025/04/04 20:24:34.635 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:35:33, a: 703, b: 703, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 703, b: 703, QH1PS: 6MveZ03txklr6VXyTBzrN4WjBCn0PAf1, c: Ci5uKDt4g, d: 20:35:33, }"] [2025/04/04 20:24:34.735 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:35:34, a: 704, b: 704, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: w2Yb, d: 20:35:34, a: 704, b: 704, QH1PS: ktD8FK1QuUGGS8NaRe6zpIp4Z41Y, }"] wait process cdc.test exit for 1-th time... [2025/04/04 20:24:34.835 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 705, b: 705, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:35, }"] [row2="{ QH1PS: TkgvJN2vqG8TByGa4SzPs2UXmqBXslP, c: 4, d: 20:35:35, a: 705, b: 705, }"] [2025/04/04 20:24:34.935 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 706, b: 706, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:36, }"] [row2="{ c: NsdI2q4rO, d: 20:35:36, a: 706, b: 706, QH1PS: yzBKPzOzPF4U4Kz2vCZFsGAmDfXmh1, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:35.035 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 707, b: 707, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:37, }"] [row2="{ c: 5YbWVt, d: 20:35:37, a: 707, b: 707, QH1PS: ku1E1vqQzaqsptuUqhacSed6ee8, }"] [2025/04/04 20:24:35.136 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 708, b: 708, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:38, }"] [row2="{ d: 20:35:38, a: 708, b: 708, QH1PS: iaQEwkDiOrFRq9IyeUg4WqTi5wlireeH, c: VWx8lKnCS, }"] [2025/04/04 20:24:35.236 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 709, b: 709, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:39, }"] [row2="{ d: 20:35:39, a: 709, b: 709, QH1PS: PNhwSSKnjP30Fsvk4zAO246Oqh5o, c: 1JgEhmPjz, }"] wait process cdc.test exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 [2025/04/04 20:24:35.336 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:35:40, a: 710, b: 710, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 710, b: 710, QH1PS: icLexQHvT8S5an2ZHyEbNVPYUf14JD, c: um, d: 20:35:40, }"] [2025/04/04 20:24:35.436 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:35:41, a: 711, b: 711, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 711, QH1PS: aOvI2WRqdO6TlQjYuJH, c: 4gO, d: 20:35:41, a: 711, }"] + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5113 0 --:--:-- --:--:-- --:--:-- 5139 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 20:24:29.077","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 20:24:35.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:24:29.077","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:24:35.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:24:29.077","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:24:35.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2025-04-04 20:24:29.077' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:24:29.077","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:24:35.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:24:29.077","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:24:35.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d93950 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118705009033224 MinTableBarrierTs:457118705271439370 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22726268-44ad-42ad-83da-cef520863f37 {"id":"22726268-44ad-42ad-83da-cef520863f37","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769466} /tidb/cdc/default/__cdc_meta__/capture/8006d62c-6d72-4459-bb3a-c30a901461b2 {"id":"8006d62c-6d72-4459-bb3a-c30a901461b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769472} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89f7d 22726268-44ad-42ad-83da-cef520863f37 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89fcc 8006d62c-6d72-4459-bb3a-c30a901461b2 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118705009033224,"min-table-barrier-ts":457118705271439370,"admin-job-type":0} /tidb/cdc/default/default/task/position/22726268-44ad-42ad-83da-cef520863f37/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8006d62c-6d72-4459-bb3a-c30a901461b2/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d93950 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118705009033224 MinTableBarrierTs:457118705271439370 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22726268-44ad-42ad-83da-cef520863f37 {"id":"22726268-44ad-42ad-83da-cef520863f37","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769466} /tidb/cdc/default/__cdc_meta__/capture/8006d62c-6d72-4459-bb3a-c30a901461b2 {"id":"8006d62c-6d72-4459-bb3a-c30a901461b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769472} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89f7d 22726268-44ad-42ad-83da-cef520863f37 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89fcc 8006d62c-6d72-4459-bb3a-c30a901461b2 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118705009033224,"min-table-barrier-ts":457118705271439370,"admin-job-type":0} /tidb/cdc/default/default/task/position/22726268-44ad-42ad-83da+ grep -q 'failed to get info:' -cef520863f37/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8006d62c-6d72-4459-bb3a-c30a901461b2/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d93950 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118705009033224 MinTableBarrierTs:457118705271439370 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22726268-44ad-42ad-83da-cef520863f37 {"id":"22726268-44ad-42ad-83da-cef520863f37","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769466} /tidb/cdc/default/__cdc_meta__/capture/8006d62c-6d72-4459-bb3a-c30a901461b2 {"id":"8006d62c-6d72-4459-bb3a-c30a901461b2","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769472} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89f7d 22726268-44ad-42ad-83da-cef520863f37 /tidb/cdc/default/__cdc_meta__/owner/22319600bee89fcc 8006d62c-6d72-4459-bb3a-c30a901461b2 /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118705009033224,"min-table-barrier-ts":457118705271439370,"admin-job-type":0} /tidb/cdc/default/default/task/position/22726268-44ad-42ad-83da-cef520863f37/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/8006d62c-6d72-4459-bb3a-c30a901461b2/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "22726268-44ad-42ad-83da-cef520863f37" | grep id + stop_tidb_cluster [2025/04/04 20:24:35.536 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 712, b: 712, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:42, }"] [row2="{ c: p4cKW, d: 20:35:42, a: 712, b: 712, QH1PS: GSbb6b8gYmM2qLGZYbU, }"] [2025/04/04 20:24:35.636 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:35:43, a: 713, b: 713, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: DMELMadh9, d: 20:35:43, a: 713, b: 713, QH1PS: aRKKrR5Dl, }"] [2025/04/04 20:24:35.737 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 714, b: 714, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:44, }"] [row2="{ a: 714, b: 714, QH1PS: RzhAHhCbJrNBuI6bL, c: pOP, d: 20:35:44, }"] + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2025-04-04 20:24:35' + echo 'sink_checkpoint_ts is 2025-04-04' 20:24:29.077 sink_checkpoint_ts is 2025-04-04 20:24:29.077 ++ date -d '2025-04-04 20:24:29.077' +%s + checkpoint_timestamp=1743769469 ++ date -d '2025-04-04 20:24:35' +%s + current_timestamp=1743769475 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:24:35 CST 2025] <<<<<< run test case ddl_reentrant success! >>>>>> [2025/04/04 20:24:35.837 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:45, a: 715, b: 715, }"] [row2="{ a: 715, b: 715, QH1PS: rHWB3C68Ivd9H0ZnRabaLIIWHGVf7oTW, c: d0KO, d: 20:35:45, }"] [2025/04/04 20:24:35.937 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:46, a: 716, b: 716, }"] [row2="{ b: 716, QH1PS: J4U2kuE, c: ZcZdS, d: 20:35:46, a: 716, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:36.037 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 717, b: 717, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:47, }"] [row2="{ a: 717, b: 717, QH1PS: enKWNhNuQCE2ydQjOsipPaj50TDukZKGg2YWRnTb12Se, c: p, d: 20:35:47, }"] [2025/04/04 20:24:36.137 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 718, b: 718, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:48, }"] [row2="{ b: 718, QH1PS: QvDaQwL2AQRQPhNxu, c: NaBDU8Jv, d: 20:35:48, a: 718, }"] [2025/04/04 20:24:36.238 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 719, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:30, a: 719, }"] [row2="{ a: 719, b: 719, QH1PS: p4xqTUdYpuPHVf, c: rNxe, d: 20:35:49, }"] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table kill_owner_with_ddl.t3 exists check diff failed 1-th time, retry later [2025/04/04 20:24:36.338 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 720, b: 720, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:50, }"] [row2="{ a: 720, b: 720, QH1PS: 4K1zdufEha59vojtry2jwcbtBSCf08WFahdgiG, c: eERsLrG, d: 20:35:50, }"] [2025/04/04 20:24:36.438 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:35:51, a: 721, b: 721, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 721, b: 721, QH1PS: M8w5rYt62, c: BlFp, d: 20:35:51, }"] [2025/04/04 20:24:36.538 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 722, b: 722, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:52, }"] [row2="{ QH1PS: PSpXJ0eN1iWu003czgKOPAiYpznUoFy, c: YlEwL, d: 20:35:52, a: 722, b: 722, }"] [2025/04/04 20:24:36.638 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 723, b: 723, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:53, }"] [row2="{ d: 20:35:53, a: 723, b: 723, QH1PS: EADC4hc, c: xRpg4djm, }"] [2025/04/04 20:24:36.738 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 724, b: 724, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:54, }"] [row2="{ a: 724, b: 724, QH1PS: 78E1c0Yic3THMYuh1NSWoQYIa5n43zDNk7XZ, c: lUD9i6, d: 20:35:54, }"] ***************** properties ***************** "recordcount"="10" "dotransactions"="false" "updateproportion"="0" "operationcount"="0" "readallfields"="true" "threadcount"="2" "mysql.db"="testSync" "mysql.port"="4000" "readproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.user"="root" "workload"="core" ********************************************** Run finished, takes 4.407514ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3585.9, Avg(us): 819, Min(us): 546, Max(us): 1692, 95th(us): 2000, 99th(us): 2000 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:36.43133+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:24:36.465168+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:36.524262+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} "cluster-id": "default" "id": "8006d62c-6d72-4459-bb3a-c30a901461b2", "cluster-id": "default" run task successfully [2025/04/04 20:24:36.839 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 725, b: 725, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:55, }"] [row2="{ QH1PS: g, c: lSBReLJ, d: 20:35:55, a: 725, b: 725, }"] [2025/04/04 20:24:36.939 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 726, b: 726, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:56, }"] [row2="{ d: 20:35:56, a: 726, b: 726, QH1PS: zrwuUAF8M152Q, c: HT, }"] [2025/04/04 20:24:37.039 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:35:57, a: 727, b: 727, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: 66G6mABva, d: 20:35:57, a: 727, b: 727, QH1PS: Rv6V3VBs0WM2DD9vQ5vvffo0ZS8rRRuD7SABBS, }"] [2025/04/04 20:24:37.139 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 728, b: 728, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:58, }"] [row2="{ b: 728, QH1PS: Y2t5HjO4c, c: qofZ3, d: 20:35:58, a: 728, }"] [2025/04/04 20:24:37.239 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 729, b: 729, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:35:59, }"] [row2="{ a: 729, b: 729, QH1PS: bZa, c: FYCRDT, d: 20:35:59, }"] + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:37 GMT < Content-Length: 1714 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0569f733-cd91-4619-a474-5a4b06d0cb66 {"id":"0569f733-cd91-4619-a474-5a4b06d0cb66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769469} /tidb/cdc/default/__cdc_meta__/capture/4669d935-7571-4836-b927-6132cf441b64 {"id":"4669d935-7571-4836-b927-6132cf441b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264dfdd 0569f733-cd91-4619-a474-5a4b06d0cb66 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264e01d 4669d935-7571-4836-b927-6132cf441b64 /tidb/cdc/default/default/upstream/7489432815991351305 {"id":7489432815991351305,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0569f733-cd91-4619-a474-5a4b06d0cb66 {"id":"0569f733-cd91-4619-a474-5a4b06d0cb66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769469} /tidb/cdc/default/__cdc_meta__/capture/4669d935-7571-4836-b927-6132cf441b64 {"id":"4669d935-7571-4836-b927-6132cf441b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264dfdd 0569f733-cd91-4619-a474-5a4b06d0cb66 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264e01d 4669d935-7571-4836-b927-6132cf441b64 /tidb/cdc/default/default/upstream/7489432815991351305 {"id":7489432815991351305,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0569f733-cd91-4619-a474-5a4b06d0cb66 {"id":"0569f733-cd91-4619-a474-5a4b06d0cb66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769469} /tidb/cdc/default/__cdc_meta__/capture/4669d935-7571-4836-b927-6132cf441b64 {"id":"4669d935-7571-4836-b927-6132cf441b64","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769474} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264dfdd 0569f733-cd91-4619-a474-5a4b06d0cb66 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c264e01d 4669d935-7571-4836-b927-6132cf441b64 /tidb/cdc/default/default/upstream/7489432815991351305 {"id":7489432815991351305,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [2025/04/04 20:24:37.340 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:00, a: 730, b: 730, }"] [row2="{ a: 730, b: 730, QH1PS: VC0sCMb6S4w6CICuIyK46Rarkv, c: zYMTWK9UcJ, d: 20:36:00, }"] [2025/04/04 20:24:37.440 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 731, b: 731, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:01, }"] [row2="{ a: 731, b: 731, QH1PS: ukE, c: aR5, d: 20:36:01, }"] [2025/04/04 20:24:37.540 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:02, a: 732, b: 732, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: 4yzwWi4L, d: 20:36:02, a: 732, b: 732, QH1PS: TapXgCOT0HOFCiGdypIvZHuUA1oikInYes, }"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [2025/04/04 20:24:37.640 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 733, b: 733, QH1PS: uGPv3x, c: 6bJtH56uw, d: 20:36:03, }"] [2025/04/04 20:24:37.740 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 734, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:04, a: 734, }"] [row2="{ QH1PS: xj0DUyauQiG04mxrcHwpjPozJhc4nloG, c: nabn, d: 20:36:04, a: 734, b: 734, }"] [Pipeline] // cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // dir [2025/04/04 20:24:37.840 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 735, b: 735, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:05, }"] [row2="{ b: 735, QH1PS: htGlr8CiAlJCKT1kXroWL7, c: SMBL, d: 20:36:05, a: 735, }"] [2025/04/04 20:24:37.941 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 736, b: 736, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:06, }"] [row2="{ b: 736, QH1PS: QbOLbvNbMa7, c: nVy87DN, d: 20:36:06, a: 736, }"] [2025/04/04 20:24:38.041 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 737, b: 737, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:07, }"] [row2="{ b: 737, QH1PS: JFEAKOQk2NMZIrbjJdEBpf, c: Bic, d: 20:36:07, a: 737, }"] [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // stage [Pipeline] } table test.t1 exists + sleep 5 check diff successfully [Pipeline] // container [2025/04/04 20:24:38.141 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 738, b: 738, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:08, }"] [row2="{ QH1PS: SrimMvO1mRY7f3Zrvq9y3tOu6rqa0HKlm, c: 2xB, d: 20:36:08, a: 738, b: 738, }"] [2025/04/04 20:24:38.241 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 739, b: 739, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:09, }"] [row2="{ b: 739, QH1PS: HzbWUZJ0sjiJ4z7jEO, c: 6dJ8JuBWQY, d: 20:36:09, a: 739, }"] [Pipeline] } nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] // withEnv [Pipeline] } [2025/04/04 20:24:38.341 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 740, b: 740, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:10, }"] [row2="{ a: 740, b: 740, QH1PS: UbxiGnX2OJgtIkXS0wLvKWgQ3isfeE, c: T15R2D, d: 20:36:10, }"] [2025/04/04 20:24:38.441 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 741, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:47, a: 741, }"] [row2="{ QH1PS: KHwCTYcbNdZXn63uTE, c: 7xT9hwEP, d: 20:36:11, a: 741, b: 741, }"] [2025/04/04 20:24:38.542 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 742, b: 742, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:12, }"] [row2="{ a: 742, b: 742, QH1PS: PcKa1K5rMpBUTh1eytMpSEx8BCaf30nUILwf9SrXtQZ, c: 1z, d: 20:36:12, }"] [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2025/04/04 20:24:38.642 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:13, a: 743, b: 743, }"] [row2="{ a: 743, b: 743, QH1PS: zyTPT4oBkIRVQFYHnQrHJmhIVzn466ZS5e3Cdw5, c: tv, d: 20:36:13, }"] [2025/04/04 20:24:38.742 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 744, b: 744, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:14, }"] [row2="{ a: 744, b: 744, QH1PS: 6n4K8dqJZu1wcbnJEzUedXzniwyMu, c: y6, d: 20:36:14, }"] wait process cdc.test exit for 1-th time... [Pipeline] // stage [Pipeline] } [2025/04/04 20:24:38.842 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 745, b: 745, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:15, }"] [row2="{ c: eVzSX, d: 20:36:15, a: 745, b: 745, QH1PS: V, }"] [2025/04/04 20:24:38.942 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 746, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:16, a: 746, }"] [row2="{ QH1PS: cTCuds0uWEZTsFyxmY7pnOMvujoLA6IE, c: an8xeqx, d: 20:36:16, a: 746, b: 746, }"] [2025/04/04 20:24:39.042 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 747, b: 747, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:17, }"] [row2="{ c: ER0iBK, d: 20:36:17, a: 747, b: 747, QH1PS: jrqPUqsLziMwbM3W4Uh9vUiPx, }"] wait process cdc.test exit for 2-th time... [2025/04/04 20:24:39.143 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 748, b: 748, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:18, }"] [row2="{ a: 748, b: 748, QH1PS: YWKbb8gxFBor6vsXjD11kcTmBvXvX8VXWRab1, c: ZTuONFPf, d: 20:36:18, }"] [2025/04/04 20:24:39.243 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 749, b: 749, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:19, }"] [row2="{ a: 749, b: 749, QH1PS: uXXqT8AA, c: 1423OSh1Nl, d: 20:36:19, }"] [2025/04/04 20:24:39.343 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 750, b: 750, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:20, }"] [row2="{ a: 750, b: 750, QH1PS: UJTYluP2XZJNtuXnr98zsV543udlIvq3M, c: BPc, d: 20:36:20, }"] [2025/04/04 20:24:39.443 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:36:21, a: 751, b: 751, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 751, b: 751, QH1PS: mkUjuMoRqjtfR4q0sWJ541FYmBE6V0, c: Ag0, d: 20:36:21, }"] [2025/04/04 20:24:39.543 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:22, a: 752, b: 752, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 752, b: 752, QH1PS: jt5K5Z, c: GUs, d: 20:36:22, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432855934127450 TiDB Cluster ID. tikv_gc_leader_uuid 658030a8348000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:10050, start at 2025-04-04 20:24:37.36068326 +0800 CST m=+2.165019299 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:37.401 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:37.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:37.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432855934127450 TiDB Cluster ID. tikv_gc_leader_uuid 658030a8348000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:10050, start at 2025-04-04 20:24:37.36068326 +0800 CST m=+2.165019299 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:37.401 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:37.400 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:37.400 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432848492275615 TiDB Cluster ID. tikv_gc_leader_uuid 658030a858c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-r-06491, pid:10131, start at 2025-04-04 20:24:37.505274277 +0800 CST m=+2.254206560 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:37.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:37.475 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:37.475 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:24:39 CST 2025] <<<<<< run test case kill_owner_with_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:39.643 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 753, b: 753, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:23, }"] [row2="{ QH1PS: E0XuUgtvyw0vdO1Xy3a5R2xLVFUCRI1, c: Sp2jeFoesa, d: 20:36:23, a: 753, b: 753, }"] [2025/04/04 20:24:39.744 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 754, b: 754, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:24, }"] [row2="{ a: 754, b: 754, QH1PS: Zl84dUfvmyH, c: QxfhR58, d: 20:36:24, }"] pass test: check health pass test: get status [2025/04/04 20:24:39.844 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:25, a: 755, b: 755, }"] [row2="{ d: 20:36:25, a: 755, b: 755, QH1PS: i57UMMshZOeZJeEU4gk0QqXSlVmtL1, c: c72, }"] [2025/04/04 20:24:39.944 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 756, b: 756, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:26, }"] [row2="{ a: 756, b: 756, QH1PS: 3bfiZR9pjbS5IAeCJTuPCU4MfALq, c: FoDBqg38H, d: 20:36:26, }"] [2025/04/04 20:24:40.044 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:27, a: 757, b: 757, }"] [row2="{ a: 757, b: 757, QH1PS: 5, c: qu7hBzC408, d: 20:36:27, }"] nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2025/04/04 20:24:40.144 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 758, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:28, a: 758, }"] [row2="{ a: 758, b: 758, QH1PS: 7BpaqLzKjxusRG4iRzP, c: nbL5, d: 20:36:28, }"] [2025/04/04 20:24:40.245 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 759, b: 759, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:29, }"] [row2="{ b: 759, QH1PS: N9Gem, c: OqB, d: 20:36:29, a: 759, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 [2025/04/04 20:24:40.345 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 760, b: 760, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:30, }"] [row2="{ a: 760, b: 760, QH1PS: tk0r9h04YcfIyMb4GnCRnUNdFNLfLbW9f0B9l9Owf, c: PnwPptFODp, d: 20:36:30, }"] [2025/04/04 20:24:40.445 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:36:31, a: 761, b: 761, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:36:31, a: 761, b: 761, QH1PS: W2AqnQ72mznfMjDVWttM1kDfSWd2Pd31BGicNY14, c: AV3, }"] [2025/04/04 20:24:40.545 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 762, b: 762, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:32, }"] [row2="{ c: cOPManxReV, d: 20:36:32, a: 762, b: 762, QH1PS: yLJlurrAkP, }"] pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s [2025/04/04 20:24:40.645 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 763, b: 763, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:33, }"] [row2="{ b: 763, QH1PS: r4WqX3BojXC, c: SIlZ, d: 20:36:33, a: 763, }"] [2025/04/04 20:24:40.745 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 764, b: 764, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:34, }"] [row2="{ b: 764, QH1PS: Ttn75MgNT8nQ4Yp4, c: SKvQ5ZwI, d: 20:36:34, a: 764, }"] [2025/04/04 20:24:40.846 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 765, b: 765, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:35, }"] [row2="{ b: 765, QH1PS: a8XoAh4hjQ96HLulGBYStFeZ8PtwOMOSU2Dj, c: sx1W, d: 20:36:35, a: 765, }"] + info='{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457118706537857026, "checkpoint_time": "2025-04-04 20:24:39.896", "error": null }' + echo '{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457118706537857026, "checkpoint_time": "2025-04-04 20:24:39.896", "error": null }' { "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457118706537857026, "checkpoint_time": "2025-04-04 20:24:39.896", "error": null } ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457118706537857026, '"checkpoint_time":' '"2025-04-04' '20:24:39.896",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457118706537857026, '"checkpoint_time":' '"2025-04-04' '20:24:39.896",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s [2025/04/04 20:24:40.946 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 766, b: 766, QH1PS: W62NQHxa59o8nGNKwq07, c: f86b, d: 20:36:36, }"] [2025/04/04 20:24:41.046 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 767, b: 767, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:37, }"] [row2="{ a: 767, b: 767, QH1PS: f5gWRsfDX1VQHyR4, c: q8wp5cEQYp, d: 20:36:37, }"] + info='{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457118706577178632, "checkpoint_time": "2025-04-04 20:24:40.046", "error": null }' + echo '{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457118706577178632, "checkpoint_time": "2025-04-04 20:24:40.046", "error": null }' { "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457118706577178632, "checkpoint_time": "2025-04-04 20:24:40.046", "error": null } ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457118706577178632, '"checkpoint_time":' '"2025-04-04' '20:24:40.046",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457118706577178632, '"checkpoint_time":' '"2025-04-04' '20:24:40.046",' '"error":' null '}' ++ jq -r .error.message start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... [2025/04/04 20:24:41.146 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 768, b: 768, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:38, }"] [row2="{ a: 768, b: 768, QH1PS: rYCXvCB8iR22bReyhvdH, c: 2k2Fz, d: 20:36:38, }"] [2025/04/04 20:24:41.246 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 769, b: 769, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:39, }"] [row2="{ c: 3Et16UVK, d: 20:36:39, a: 769, b: 769, QH1PS: Y, }"] [2025/04/04 20:24:41.346 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:40, a: 770, b: 770, }"] [row2="{ QH1PS: NOumwW3VDaoGEkKBRg, c: xGi, d: 20:36:40, a: 770, b: 770, }"] + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457118706616762372, "checkpoint_time": "2025-04-04 20:24:40.197", "error": null }' + echo '{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457118706616762372, "checkpoint_time": "2025-04-04 20:24:40.197", "error": null }' { "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457118706616762372, "checkpoint_time": "2025-04-04 20:24:40.197", "error": null } ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457118706616762372, '"checkpoint_time":' '"2025-04-04' '20:24:40.197",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457118706616762372, '"checkpoint_time":' '"2025-04-04' '20:24:40.197",' '"error":' null '}' ++ jq -r .error.message 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:24:41.447 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 771, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:41, a: 771, }"] [row2="{ c: Z, d: 20:36:41, a: 771, b: 771, QH1PS: vmwx1vE0OLEOdJKxcZtCq6B5R1JoGOZw8wA9AV2, }"] [2025/04/04 20:24:41.547 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 772, b: 772, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:42, }"] [row2="{ a: 772, b: 772, QH1PS: jncCWxfH66LXTTAOBFMZcOU, c: j, d: 20:36:42, }"] + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457118706656083982, "checkpoint_time": "2025-04-04 20:24:40.347", "error": null }' + echo '{ "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457118706656083982, "checkpoint_time": "2025-04-04 20:24:40.347", "error": null }' { "upstream_id": 7489432815991351305, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457118706656083982, "checkpoint_time": "2025-04-04 20:24:40.347", "error": null } ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457118706656083982, '"checkpoint_time":' '"2025-04-04' '20:24:40.347",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432815991351305, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457118706656083982, '"checkpoint_time":' '"2025-04-04' '20:24:40.347",' '"error":' null '}' ++ jq -r .error.message VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432760158627123 TiDB Cluster ID. tikv_gc_leader_uuid 658030aa784000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:10139, start at 2025-04-04 20:24:39.666699279 +0800 CST m=+26.350937094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:39.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:39.649 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:39.649 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432760158627123 TiDB Cluster ID. tikv_gc_leader_uuid 658030aa784000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:10139, start at 2025-04-04 20:24:39.666699279 +0800 CST m=+26.350937094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:39.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:39.649 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:39.649 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432759965283451 TiDB Cluster ID. tikv_gc_leader_uuid 65803094714000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:10208, start at 2025-04-04 20:24:17.135078096 +0800 CST m=+3.730010145 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:17.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:17.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:17.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [2025/04/04 20:24:41.647 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 773, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:43, a: 773, }"] [row2="{ c: ouG1lU, d: 20:36:43, a: 773, b: 773, QH1PS: 3B1pgZuLUuHu8Tafy3UauBu93Rc3lBS, }"] [2025/04/04 20:24:41.747 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:44, a: 774, b: 774, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 774, b: 774, QH1PS: tunv75POoxWFDff3ulDcGovlMhJ8fauKclJG4RcJ, c: Qi8Thi, d: 20:36:44, }"] [2025/04/04 20:24:41.847 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 775, b: 775, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:45, }"] [row2="{ c: fXbw, d: 20:36:45, a: 775, b: 775, QH1PS: RDhOkbBaA4RSJQPqAyha1gHsGnZQT5TdNmvrKSXdP, }"] + message=null + [[ ! null =~ null ]] run task successfully TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:24:41.948 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:46, a: 776, b: 776, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 776, b: 776, QH1PS: NJO8VLz8xNf1Hfk1XjbU3znfyjOvnTlE5eI9dw, c: VMwMQ4, d: 20:36:46, }"] [2025/04/04 20:24:42.048 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 777, b: 777, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:47, }"] [row2="{ a: 777, b: 777, QH1PS: 6vCm4x2ha8T, c: u, d: 20:36:47, }"] [2025/04/04 20:24:42.148 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 778, b: 778, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:48, }"] [row2="{ a: 778, b: 778, QH1PS: 1gUF3lG, c: jNH4el, d: 20:36:48, }"] [2025/04/04 20:24:42.248 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:36:49, a: 779, b: 779, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 779, b: 779, QH1PS: EQAa5k, c: YU4InlG, d: 20:36:49, }"] [2025/04/04 20:24:42.348 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:50, a: 780, b: 780, }"] [row2="{ b: 780, QH1PS: qLQMtr, c: BjMwRVlOsL, d: 20:36:50, a: 780, }"] [2025/04/04 20:24:42.449 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:52, a: 782, b: 782, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: HwXOcV7y, d: 20:36:52, a: 782, b: 782, QH1PS: oZdOn6oF6nMgxOHm0osfNw7sQP9Jdofxa0FSI5mPED, }"] [2025/04/04 20:24:42.549 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:53, a: 783, b: 783, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 783, QH1PS: PUpTYuu2nq92Ennr7Z9WY0, c: 8, d: 20:36:53, a: 783, }"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' {"level":"warn","ts":"2025-04-04T20:24:42.432449+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002104a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test {"level":"warn","ts":"2025-04-04T20:24:42.466808+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:24:42.525279+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002005500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ***************** properties ***************** "mysql.user"="root" "operationcount"="0" "readproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readallfields"="true" "workload"="core" "dotransactions"="false" "recordcount"="20" "mysql.db"="changefeed_auto_stop_1" "scanproportion"="0" "insertproportion"="0" "threadcount"="4" "updateproportion"="0" ********************************************** Run finished, takes 10.079119ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3643.0, Avg(us): 1882, Min(us): 965, Max(us): 4552, 95th(us): 5000, 99th(us): 5000 [2025/04/04 20:24:42.649 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:54, a: 784, b: 784, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 784, b: 784, QH1PS: 8PsdKptjt01, c: nnfQSb6, d: 20:36:54, }"] [2025/04/04 20:24:42.749 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 785, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:55, a: 785, }"] [row2="{ d: 20:36:55, a: 785, b: 785, QH1PS: Vdje5naYb54ISI7jLhKfiWwo1L, c: 2YzIBnXe, }"] [2025/04/04 20:24:42.849 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:56, a: 786, b: 786, }"] [row2="{ c: l5WWg3, d: 20:36:56, a: 786, b: 786, QH1PS: zVLS5, }"] wait process cdc.test exit for 1-th time... [2025/04/04 20:24:42.949 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:57, a: 787, b: 787, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 787, b: 787, QH1PS: 3BweQ2I4I5CbS4adwZPH3TQzWYYLjwwUZsHAoUP, c: oEA, d: 20:36:57, }"] [2025/04/04 20:24:43.049 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:36:58, a: 788, b: 788, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ QH1PS: cQYt17mAz920jMYxG6j3KQsns, c: 4lQOun, d: 20:36:58, a: 788, b: 788, }"] [2025/04/04 20:24:43.150 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 789, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:36:59, a: 789, }"] [row2="{ a: 789, b: 789, QH1PS: T8h4ncARJfseDfjlIcIvGlGqotYA8adKrLm95vBQ6, c: Yyoo, d: 20:36:59, }"] [2025/04/04 20:24:43.250 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:00, a: 790, b: 790, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 790, b: 790, QH1PS: y7VVilwaGCrakL3, c: z8FRmC, d: 20:37:00, }"] [2025/04/04 20:24:43.350 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 791, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:01, a: 791, }"] [row2="{ c: YVufzhmYz, d: 20:37:01, a: 791, b: 791, QH1PS: 2TQOua43WWOJ9n9EW, }"] Verifying downstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 21439 0 --:--:-- --:--:-- --:--:-- 22090 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:24:42.427","puller_resolved_ts":"2025-04-04 20:24:35.628","last_synced_ts":"2025-04-04 20:24:36.127","now_ts":"2025-04-04 20:24:43.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:24:42.427","puller_resolved_ts":"2025-04-04' '20:24:35.628","last_synced_ts":"2025-04-04' '20:24:36.127","now_ts":"2025-04-04' '20:24:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:24:42.427","puller_resolved_ts":"2025-04-04' '20:24:35.628","last_synced_ts":"2025-04-04' '20:24:36.127","now_ts":"2025-04-04' '20:24:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [2025/04/04 20:24:43.450 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 792, b: 792, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:02, }"] [row2="{ b: 792, QH1PS: tiw1S8r1Ys60pOCq9szN2G4xQlLTdQeVga7cIyO4mRk6J, c: 9fA, d: 20:37:02, a: 792, }"] [2025/04/04 20:24:43.550 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 793, b: 793, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:03, }"] [row2="{ QH1PS: 6QpdiU8, c: HKqFL6bC, d: 20:37:03, a: 793, b: 793, }"] wait process cdc.test exit for 2-th time... ***************** properties ***************** "mysql.user"="root" "recordcount"="20" "mysql.host"="127.0.0.1" "scanproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "dotransactions"="false" "workload"="core" "insertproportion"="0" "updateproportion"="0" "readproportion"="0" "mysql.db"="changefeed_auto_stop_2" "threadcount"="4" ********************************************** [Fri Apr 4 20:24:43 CST 2025] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.2401124013.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:43.650 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:04, a: 794, b: 794, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 794, b: 794, QH1PS: 95B5HumcFMnQvpti5Ua2QiC, c: ORMGN, d: 20:37:04, }"] [2025/04/04 20:24:43.751 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 795, b: 795, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:05, }"] [row2="{ a: 795, b: 795, QH1PS: 6ksmcGZOx7K6NCpUeHxtbgju9yVOOe6CCm2at0X1, c: nu8qd, d: 20:37:05, }"] [2025/04/04 20:24:43.851 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 796, b: 796, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:06, }"] [row2="{ a: 796, b: 796, QH1PS: 2TNC, c: z4N5DDJiYD, d: 20:37:06, }"] Run finished, takes 10.314003ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3247.0, Avg(us): 1915, Min(us): 1065, Max(us): 4088, 95th(us): 5000, 99th(us): 5000 [2025/04/04 20:24:43.951 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 797, b: 797, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:07, }"] [row2="{ a: 797, b: 797, QH1PS: toyfRalITIiI1lbuKuaSqCm1iY8e9nYZC1ZP3rHT, c: gxJ8VjU, d: 20:37:07, }"] [2025/04/04 20:24:44.051 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:08, a: 798, b: 798, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ d: 20:37:08, a: 798, b: 798, QH1PS: SdzFB3SGaKjnM3D78egMe0, c: McOW, }"] wait process cdc.test exit for 3-th time... table test.simple1 exists running list_changefeed [2025/04/04 20:24:44.151 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 799, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:09, a: 799, }"] [row2="{ a: 799, b: 799, QH1PS: Wv8Psxjsx4LqkUGRcjmBDqcngbNoLguw, c: I, d: 20:37:09, }"] [2025/04/04 20:24:44.252 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:10, a: 800, b: 800, }"] [2025/04/04 20:24:44.352 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:37:11, a: 801, b: 801, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later pass test: list changefeed [2025/04/04 20:24:44.452 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 802, b: 802, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:12, }"] [2025/04/04 20:24:44.552 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 803, b: 803, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:13, }"] [2025/04/04 20:24:44.652 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 804, b: 804, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:14, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:24:44.752 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 805, b: 805, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:15, }"] [2025/04/04 20:24:44.853 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 806, b: 806, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:16, }"] ***************** properties ***************** "insertproportion"="0" "threadcount"="4" "operationcount"="0" "dotransactions"="false" "scanproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "mysql.port"="4000" "recordcount"="20" "mysql.user"="root" "mysql.db"="changefeed_auto_stop_3" "readproportion"="0" "workload"="core" "updateproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.215922ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3379.3, Avg(us): 1906, Min(us): 1067, Max(us): 4247, 95th(us): 5000, 99th(us): 5000 [2025/04/04 20:24:44.953 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 807, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:17, a: 807, }"] [2025/04/04 20:24:45.053 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:18, a: 808, b: 808, }"] [2025/04/04 20:24:45.153 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 809, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:19, a: 809, }"] [2025/04/04 20:24:45.253 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 810, b: 810, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:20, }"] [row2="{ a: 810, b: 810, QH1PS: Ke0EvQYdEfxeTbGxB7opv3oDLTplU, c: df, d: 20:37:20, }"] [2025/04/04 20:24:45.354 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:21, a: 811, b: 811, }"] [row2="{ a: 811, b: 811, QH1PS: jSEs6KcdqugGvH7ERtuiro02FOyMCf86AoSBPaDjUBb4a, c: EDRO, d: 20:37:21, }"] [2025/04/04 20:24:45.454 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:22, a: 812, b: 812, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 812, b: 812, QH1PS: xPi, c: OZJaNQKx, d: 20:37:22, }"] [2025/04/04 20:24:45.554 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 813, b: 813, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:23, }"] [row2="{ a: 813, b: 813, QH1PS: aYssadMaZwqgSkZ5ZOmUy3NAgQBmk, c: cwtbXlU9SW, d: 20:37:23, }"] [2025/04/04 20:24:45.654 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 814, b: 814, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:24, }"] [row2="{ a: 814, b: 814, QH1PS: NXyNf, c: 2V, d: 20:37:24, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:24:45.754 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:25, a: 815, b: 815, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ QH1PS: SGfyfiXSVuH3V, c: L0WakjQX, d: 20:37:25, a: 815, b: 815, }"] [2025/04/04 20:24:45.854 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 816, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:26, a: 816, }"] [row2="{ a: 816, b: 816, QH1PS: TpR2lyK, c: UD4, d: 20:37:26, }"] running get_changefeed pass test: get changefeed [2025/04/04 20:24:45.955 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:27, a: 817, b: 817, }"] [row2="{ a: 817, b: 817, QH1PS: C09IWdOHiA, c: SbQxsGMuY, d: 20:37:27, }"] [2025/04/04 20:24:46.055 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:37:28, a: 818, b: 818, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: YtjRutA, d: 20:37:28, a: 818, b: 818, QH1PS: xFAphBZsg6iwhxw484MX, }"] [2025/04/04 20:24:46.155 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 819, b: 819, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:29, }"] [row2="{ b: 819, QH1PS: JLPplP6vNwDnL, c: kqz5l, d: 20:37:29, a: 819, }"] check_redo_resolved_ts 173e4daf-f096-499f-b960-35825eb11c1a 457118707834421254 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/173e4daf-f096-499f-b960-35825eb11c1a/meta global resolved ts 457118702368981000 not forward to 457118707834421254 run task failed 1-th time, retry later ***************** properties ***************** "operationcount"="0" "requestdistribution"="uniform" "readallfields"="true" "readproportion"="0" "workload"="core" "updateproportion"="0" "mysql.user"="root" "scanproportion"="0" "insertproportion"="0" "dotransactions"="false" "mysql.db"="changefeed_auto_stop_4" "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.port"="4000" "threadcount"="4" ********************************************** Run finished, takes 10.277646ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3882.3, Avg(us): 1955, Min(us): 940, Max(us): 5092, 95th(us): 6000, 99th(us): 6000 [Fri Apr 4 20:24:45 CST 2025] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1159511597.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:24:46.255 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 820, b: 820, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:32, }"] [2025/04/04 20:24:46.355 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:37:33, a: 821, b: 821, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:24:46.456 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 822, b: 822, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:34, }"] [2025/04/04 20:24:46.556 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 823, b: 823, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:35, }"] [2025/04/04 20:24:46.656 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:37:36, a: 824, b: 824, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:46.756 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 825, b: 825, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:37, }"] [2025/04/04 20:24:46.856 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 826, b: 826, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:38, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:46 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5735d364-2843-48dd-b89b-b765701cd2aa {"id":"5735d364-2843-48dd-b89b-b765701cd2aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769484} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c232eb25 5735d364-2843-48dd-b89b-b765701cd2aa /tidb/cdc/default/default/upstream/7489432760158627123 {"id":7489432760158627123,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5735d364-2843-48dd-b89b-b765701cd2aa {"id":"5735d364-2843-48dd-b89b-b765701cd2aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769484} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c232eb25 5735d364-2843-48dd-b89b-b765701cd2aa /tidb/cdc/default/default/upstream/7489432760158627123 {"id":7489432760158627123,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5735d364-2843-48dd-b89b-b765701cd2aa {"id":"5735d364-2843-48dd-b89b-b765701cd2aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769484} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c232eb25 5735d364-2843-48dd-b89b-b765701cd2aa /tidb/cdc/default/default/upstream/7489432760158627123 {"id":7489432760158627123,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.24333.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check_redo_resolved_ts 173e4daf-f096-499f-b960-35825eb11c1a 457118707834421254 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/173e4daf-f096-499f-b960-35825eb11c1a/meta global resolved ts 457118702368981000 not forward to 457118707834421254 run task failed 2-th time, retry later [2025/04/04 20:24:46.956 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:37:39, a: 827, b: 827, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:47.057 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 828, b: 828, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:40, }"] [2025/04/04 20:24:47.157 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:41, a: 829, b: 829, }"] Create changefeed successfully! ID: 267985b3-12d3-4245-aa54-7364f3cdcd59 Info: {"upstream_id":7489432760158627123,"namespace":"default","id":"267985b3-12d3-4245-aa54-7364f3cdcd59","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:24:47.207489786+08:00","start_ts":457118708426080262,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118708426080262,"checkpoint_ts":457118708426080262,"checkpoint_time":"2025-04-04 20:24:47.099"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... running pause_changefeed [2025/04/04 20:24:47.257 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:37:42, a: 830, b: 830, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: ttA, d: 20:37:42, a: 830, b: 830, QH1PS: lr7megvluLlusnXWVVUapA35cfKCuNgc9w4dJb, }"] [2025/04/04 20:24:47.357 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 831, b: 831, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:43, }"] [row2="{ b: 831, QH1PS: LSr53KOsaX2vL6x5d3LOIGBplbHRLVAFKBKtV02n3Gl, c: QWLIQY, d: 20:37:43, a: 831, }"] [2025/04/04 20:24:47.457 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 832, b: 832, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:44, }"] [row2="{ a: 832, b: 832, QH1PS: 0M38itbqdPrun, c: EPYNi, d: 20:37:44, }"] [2025/04/04 20:24:47.558 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:45, a: 833, b: 833, }"] [row2="{ c: arSzfwd, d: 20:37:45, a: 833, b: 833, QH1PS: lZzWXlqYpK, }"] [2025/04/04 20:24:47.658 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:46, a: 834, b: 834, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: UP, d: 20:37:46, a: 834, b: 834, QH1PS: 8jIZItFOgjPdJMFrFs9uTf7xub7TvoZcJYMDQfXbNm, }"] [2025/04/04 20:24:47.758 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 835, b: 835, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:47, }"] [row2="{ QH1PS: pAmhAcqXDGhLn3K7zcJBTPFP3VbXIwpb2Z1f2J0HTZ, c: dbJFHbjjH8, d: 20:37:47, a: 835, b: 835, }"] [2025/04/04 20:24:47.858 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:48, a: 836, b: 836, }"] [row2="{ d: 20:37:48, a: 836, b: 836, QH1PS: 0TDB1ejosb, c: 3BFI, }"] [2025/04/04 20:24:47.958 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:37:49, a: 837, b: 837, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 837, QH1PS: Lw8, c: FD, d: 20:37:49, a: 837, }"] [2025/04/04 20:24:48.058 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 838, b: 838, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:50, }"] [row2="{ d: 20:37:50, a: 838, b: 838, QH1PS: TxR8MlR8eVPMr497v6r2y5EtTOdDAu, c: hStomfa, }"] [2025/04/04 20:24:48.158 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 839, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:51, a: 839, }"] [row2="{ QH1PS: iTBySVGxItuQLKC80u4T6EjAGhUEqZwHVxblKx9mEH, c: FkuDNkt, d: 20:37:51, a: 839, b: 839, }"] [2025/04/04 20:24:48.259 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 840, b: 840, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:52, }"] [row2="{ QH1PS: 6uFbP7X9EP5QRprAMLM2kqAmzE5kRaQ, c: YlXUs0Mq, d: 20:37:52, a: 840, b: 840, }"] [2025/04/04 20:24:48.359 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 841, b: 841, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:53, }"] [row2="{ a: 841, b: 841, QH1PS: tFp8zX25Xt9EA1n7YZjNOYAfc8b, c: m34mcz4kQ0, d: 20:37:53, }"] + set +x pass test: pause changefeed [2025/04/04 20:24:48.459 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:37:54, a: 842, b: 842, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 842, b: 842, QH1PS: C6Vt8nKbOJa7LKW6uDt6oyYq1qbvdxXZ6SV5QP5Qj0J, c: cD7u, d: 20:37:54, }"] [2025/04/04 20:24:48.559 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 843, b: 843, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:55, }"] [row2="{ a: 843, b: 843, QH1PS: jigctNDxdjQehqYVlG3a, c: tm8Q, d: 20:37:55, }"] [2025/04/04 20:24:48.659 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 844, b: 844, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:56, }"] [row2="{ a: 844, b: 844, QH1PS: I5Ny5, c: X4KLtN, d: 20:37:56, }"] [2025/04/04 20:24:48.760 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:57, a: 845, b: 845, }"] [row2="{ a: 845, b: 845, QH1PS: 3kNiqaN0utuGQamh2S3oHGelFS3FIhsf7WWS1, c: WJVu5cV, d: 20:37:57, }"] [2025/04/04 20:24:48.860 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:37:58, a: 846, b: 846, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ QH1PS: NE, c: Lqld7, d: 20:37:58, a: 846, b: 846, }"] [2025/04/04 20:24:48.960 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 847, b: 847, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:37:59, }"] [row2="{ a: 847, b: 847, QH1PS: ERdHIPrWSum, c: nA27DxH, d: 20:37:59, }"] start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:49.060 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:38:00, a: 848, b: 848, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 848, b: 848, QH1PS: iXHHzKkCtdu3, c: Bb, d: 20:38:00, }"] [2025/04/04 20:24:49.160 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 849, b: 849, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:01, }"] [row2="{ a: 849, b: 849, QH1PS: W20eRsL, c: Hccyc7, d: 20:38:01, }"] [2025/04/04 20:24:49.261 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 850, b: 850, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:03, }"] [2025/04/04 20:24:49.361 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:04, a: 851, b: 851, }"] [2025/04/04 20:24:49.461 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:38:05, a: 852, b: 852, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:49 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e589bb59-e215-4e78-ba2b-2ec509e8ab5c {"id":"e589bb59-e215-4e78-ba2b-2ec509e8ab5c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2809236 e589bb59-e215-4e78-ba2b-2ec509e8ab5c /tidb/cdc/default/default/upstream/7489432855934127450 {"id":7489432855934127450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e589bb59-e215-4e78-ba2b-2ec509e8ab5c {"id":"e589bb59-e215-4e78-ba2b-2ec509e8ab5c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2809236 e589bb59-e215-4e78-ba2b-2ec509e8ab5c /tidb/cdc/default/default/upstream/7489432855934127450 {"id":7489432855934127450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e589bb59-e215-4e78-ba2b-2ec509e8ab5c {"id":"e589bb59-e215-4e78-ba2b-2ec509e8ab5c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2809236 e589bb59-e215-4e78-ba2b-2ec509e8ab5c /tidb/cdc/default/default/upstream/7489432855934127450 {"id":7489432855934127450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:24:49 CST 2025] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1164111643.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... running update_changefeed [2025/04/04 20:24:49.561 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:38:06, a: 853, b: 853, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:24:49.661 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 854, b: 854, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:07, }"] pass test: update changefeed [2025/04/04 20:24:49.761 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:38:08, a: 855, b: 855, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:24:49.862 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 856, b: 856, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:09, }"] [2025/04/04 20:24:49.962 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 857, b: 857, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:10, }"] ***************** properties ***************** "recordcount"="1000" "scanproportion"="0" "mysql.port"="4000" "dotransactions"="false" "workload"="core" "operationcount"="0" "readproportion"="0" "threadcount"="2" "readallfields"="true" "table"="a1" "insertproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** [2025/04/04 20:24:50.062 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 858, b: 858, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:11, }"] [2025/04/04 20:24:50.162 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:38:12, a: 859, b: 859, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] Run finished, takes 510.388051ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 2012.1, Avg(us): 990, Min(us): 757, Max(us): 13295, 95th(us): 2000, 99th(us): 2000 [2025/04/04 20:24:50.262 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 860, b: 860, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:13, }"] [row2="{ a: 860, b: 860, QH1PS: WRIAgOP5LYovB1hw4jJhcQWnBeOtyvX, c: COY, d: 20:38:13, }"] [2025/04/04 20:24:50.363 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 861, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:14, a: 861, }"] [row2="{ d: 20:38:14, a: 861, b: 861, QH1PS: BYy20q4FJXBsJKQzMdjFdh4Y, c: Ex1C6OWSjA, }"] [2025/04/04 20:24:50.463 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:15, a: 862, b: 862, }"] [row2="{ a: 862, b: 862, QH1PS: Y5x96JSCXnTgaGTHhgk7, c: Q72Q, d: 20:38:15, }"] [2025/04/04 20:24:50.563 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:38:16, a: 863, b: 863, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:38:16, a: 863, b: 863, QH1PS: ZzatCdn0wXmfdsn7g4qSrTNzFBTGPTXJ69NqjK, c: 15mP, }"] [2025/04/04 20:24:50.663 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:17, a: 864, b: 864, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: TQ0IWO9, d: 20:38:17, a: 864, b: 864, QH1PS: PpzjoQs, }"] ***************** properties ***************** "mysql.port"="4000" "table"="a2" "scanproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "mysql.user"="root" "operationcount"="0" "dotransactions"="false" "recordcount"="1000" "workload"="core" "readproportion"="0" "threadcount"="2" "mysql.db"="multi_rocks" "readallfields"="true" "insertproportion"="0" ********************************************** nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully wait process cdc.test exit for 1-th time... [2025/04/04 20:24:50.763 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:18, a: 865, b: 865, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:38:18, a: 865, b: 865, QH1PS: e1KrpuCPxE0OPG0sz, c: SBtbl3K05, }"] [2025/04/04 20:24:50.863 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:19, a: 866, b: 866, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 866, b: 866, QH1PS: uUaRxLZx4cBNYZulHsd2yyBWqs9XQzTwnllIsEIc, c: CekKRjnP, d: 20:38:19, }"] [2025/04/04 20:24:50.964 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 867, b: 867, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:20, }"] [row2="{ b: 867, QH1PS: HJWR48f56q2fbqFbpYNAQHDoDpclXO8ccW5Wb0P, c: PkKbgvFoW, d: 20:38:20, a: 867, }"] Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432901003976052 TiDB Cluster ID. tikv_gc_leader_uuid 658030b3880000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:32556, start at 2025-04-04 20:24:48.963451667 +0800 CST m=+2.443247303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:48.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:48.975 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:48.975 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432901003976052 TiDB Cluster ID. tikv_gc_leader_uuid 658030b3880000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:32556, start at 2025-04-04 20:24:48.963451667 +0800 CST m=+2.443247303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:48.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:48.975 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:48.975 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432903071191360 TiDB Cluster ID. tikv_gc_leader_uuid 658030b37b40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-h-tfwlb, pid:32626, start at 2025-04-04 20:24:48.880562704 +0800 CST m=+2.306145446 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:48.888 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:48.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:48.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } running resume_changefeed check_redo_resolved_ts 173e4daf-f096-499f-b960-35825eb11c1a 457118707834421254 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/173e4daf-f096-499f-b960-35825eb11c1a/meta [2025/04/04 20:24:51.064 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 868, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:21, a: 868, }"] [row2="{ d: 20:38:21, a: 868, b: 868, QH1PS: ygnkhNyQ686YhBWa2BpFPb5u0yh, c: s, }"] [2025/04/04 20:24:51.164 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:38:22, a: 869, b: 869, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: fxoWmTqbwP, d: 20:38:22, a: 869, b: 869, QH1PS: TBfCffzdEpvfgaP4sHUdpXbZZIhoeCxRCVTaNhicM, }"] Run finished, takes 545.965859ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1901.0, Avg(us): 1063, Min(us): 797, Max(us): 19811, 95th(us): 2000, 99th(us): 2000 wait process cdc.test exit for 2-th time... run task successfully [2025/04/04 20:24:51.264 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 870, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:24, a: 870, }"] [row2="{ a: 870, b: 870, QH1PS: 7cw5nqUc7mzL24g, c: sTkhMekGP, d: 20:38:24, }"] [2025/04/04 20:24:51.364 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:38:25, a: 871, b: 871, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 871, b: 871, QH1PS: zWvJuI9G41V, c: vcQw, d: 20:38:25, }"] [2025/04/04 20:24:51.464 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:26, a: 872, b: 872, }"] [row2="{ a: 872, b: 872, QH1PS: bDLnyXbbDa093mQy9GhK8feXeDD2Z, c: QQOxTI7T, d: 20:38:26, }"] [2025/04/04 20:24:51.565 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 873, b: 873, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:27, }"] [row2="{ a: 873, b: 873, QH1PS: IKqhDtdu7XCoS31Me4pdQfgr8J28JmAptDv6VMjfj, c: dczeqw, d: 20:38:27, }"] [2025/04/04 20:24:51.665 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 874, b: 874, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:28, }"] [row2="{ a: 874, b: 874, QH1PS: jZS4HYjBZGefCquAttanVkwdG, c: gk, d: 20:38:28, }"] ***************** properties ***************** "requestdistribution"="uniform" "threadcount"="2" "scanproportion"="0" "readproportion"="0" "recordcount"="1000" "insertproportion"="0" "mysql.db"="multi_rocks" "operationcount"="0" "dotransactions"="false" "readallfields"="true" "table"="a3" "mysql.port"="4000" "workload"="core" "mysql.user"="root" "mysql.host"="127.0.0.1" "updateproportion"="0" ********************************************** [2025/04/04 20:24:51.765 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 875, b: 875, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:29, }"] [row2="{ a: 875, b: 875, QH1PS: tTF95, c: 06CUFIzRM7, d: 20:38:29, }"] [2025/04/04 20:24:51.865 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 876, b: 876, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:30, }"] [row2="{ d: 20:38:30, a: 876, b: 876, QH1PS: IY2wcFeZPZFGz0Mh9zuOHMvHRYDM8DHbBJfe, c: 9UdOu, }"] [2025/04/04 20:24:51.966 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 877, b: 877, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:31, }"] [row2="{ d: 20:38:31, a: 877, b: 877, QH1PS: jAf9rvH7GCxuc9QbyyXPuetB7hEmMMd6cACLkRd, c: P0jeKVlt, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_expire_capture [Fri Apr 4 20:24:51 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.64166418.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:52 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e39265c2-2511-41bf-bf5c-02ed35ee404f {"id":"e39265c2-2511-41bf-bf5c-02ed35ee404f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769489} /tidb/cdc/default/__cdc_meta__/capture/e589bb59-e215-4e78-ba2b-2ec509e8ab5c {"id":"e589bb59-e215-4e78-ba2b-2ec509e8ab5c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2809236 e589bb59-e215-4e78-ba2b-2ec509e8ab5c /tidb/cdc/default/__cdc_meta__/owner/22319600c2809269 e39265c2-2511-41bf-bf5c-02ed35ee404f /tidb/cdc/default/default/upstream/7489432855934127450 {"id":7489432855934127450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e39265c2-2511-41bf-bf5c-02ed35ee404f {"id":"e39265c2-2511-41bf-bf5c-02ed35ee404f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769489} /tidb/cdc/default/__cdc_meta__/capture/e589bb59-e215-4e78-ba2b-2ec509e8ab5c {"id":"e589bb59-e215-4e78-ba2b-2ec509e8ab5c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2809236 e589bb59-e215-4e78-ba2b-2ec509e8ab5c /tidb/cdc/default/__cdc_meta__/owner/22319600c2809269 e39265c2-2511-41bf-bf5c-02ed35ee404f /tidb/cdc/default/default/upstream/7489432855934127450 {"id":7489432855934127450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e39265c2-2511-41bf-bf5c-02ed35ee404f {"id":"e39265c2-2511-41bf-bf5c-02ed35ee404f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769489} /tidb/cdc/default/__cdc_meta__/capture/e589bb59-e215-4e78-ba2b-2ec509e8ab5c {"id":"e589bb59-e215-4e78-ba2b-2ec509e8ab5c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769486} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2809236 e589bb59-e215-4e78-ba2b-2ec509e8ab5c /tidb/cdc/default/__cdc_meta__/owner/22319600c2809269 e39265c2-2511-41bf-bf5c-02ed35ee404f /tidb/cdc/default/default/upstream/7489432855934127450 {"id":7489432855934127450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:24:52.066 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 878, b: 878, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:32, }"] [row2="{ QH1PS: VkDS2L5tz43Lu, c: dLjSam, d: 20:38:32, a: 878, b: 878, }"] [2025/04/04 20:24:52.166 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 879, b: 879, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:33, }"] [row2="{ a: 879, b: 879, QH1PS: baBY6c1e6Yq5ZrmWvtRGaYAnbNsNr0Xj86t1, c: S, d: 20:38:33, }"] check_changefeed_state http://127.0.0.1:2379 4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c -s Run finished, takes 527.665407ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1967.7, Avg(us): 1027, Min(us): 768, Max(us): 19373, 95th(us): 2000, 99th(us): 2000 [2025/04/04 20:24:52.266 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 880, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:34, a: 880, }"] [row2="{ QH1PS: 6w3wNE, c: 8CiUwS, d: 20:38:34, a: 880, b: 880, }"] [2025/04/04 20:24:52.366 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:35, a: 881, b: 881, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 881, QH1PS: OBEEzUJD5MUpI5D, c: jKco3V8my, d: 20:38:35, a: 881, }"] [2025/04/04 20:24:52.466 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:36, a: 882, b: 882, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ c: 1C61jm5, d: 20:38:36, a: 882, b: 882, QH1PS: 6COcwGdGzbWtkYLwLsv7Edfmc26B, }"] + info='{ "upstream_id": 7489432855934127450, "namespace": "default", "id": "4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c", "state": "normal", "checkpoint_tso": 457118706992414721, "checkpoint_time": "2025-04-04 20:24:41.630", "error": null }' + echo '{ "upstream_id": 7489432855934127450, "namespace": "default", "id": "4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c", "state": "normal", "checkpoint_tso": 457118706992414721, "checkpoint_time": "2025-04-04 20:24:41.630", "error": null }' { "upstream_id": 7489432855934127450, "namespace": "default", "id": "4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c", "state": "normal", "checkpoint_tso": 457118706992414721, "checkpoint_time": "2025-04-04 20:24:41.630", "error": null } ++ echo '{' '"upstream_id":' 7489432855934127450, '"namespace":' '"default",' '"id":' '"4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c",' '"state":' '"normal",' '"checkpoint_tso":' 457118706992414721, '"checkpoint_time":' '"2025-04-04' '20:24:41.630",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489432855934127450, '"namespace":' '"default",' '"id":' '"4c38ae55-b2f7-4d7f-93bf-74ca9a4f379c",' '"state":' '"normal",' '"checkpoint_tso":' 457118706992414721, '"checkpoint_time":' '"2025-04-04' '20:24:41.630",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later pass test: resume changefeed wait process cdc.test exit for 2-th time... [2025/04/04 20:24:52.567 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:38:37, a: 883, b: 883, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 883, b: 883, QH1PS: NbVkAkrIa7EkKt0ZikHSedVR689, c: uKF2MxxZb, d: 20:38:37, }"] [2025/04/04 20:24:52.667 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 884, b: 884, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:38, }"] [row2="{ a: 884, b: 884, QH1PS: cDq5SgpgVrv04S1jpcjmfAfOLZao, c: 9h, d: 20:38:38, }"] + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34019.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/04 20:24:52.767 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 885, b: 885, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:39, }"] [row2="{ c: VOzeak, d: 20:38:39, a: 885, b: 885, QH1PS: T1IPJhJAsb51dsp2S3xLTdeosxc8Fl7OWbadkipXR0fq0, }"] [2025/04/04 20:24:52.867 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:38:40, a: 886, b: 886, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: n, d: 20:38:40, a: 886, b: 886, QH1PS: YGJppOBQreq5D6aOrR, }"] [2025/04/04 20:24:52.967 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 887, b: 887, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:41, }"] [row2="{ a: 887, b: 887, QH1PS: FP, c: EmNB, d: 20:38:41, }"] ***************** properties ***************** "requestdistribution"="uniform" "recordcount"="1000" "dotransactions"="false" "mysql.user"="root" "operationcount"="0" "readproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "table"="a4" "threadcount"="2" "mysql.port"="4000" "workload"="core" "updateproportion"="0" "scanproportion"="0" "mysql.db"="multi_rocks" "readallfields"="true" ********************************************** wait process cdc.test exit for 3-th time... [2025/04/04 20:24:53.068 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 888, b: 888, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:42, }"] [row2="{ a: 888, b: 888, QH1PS: ywQq1EsVbWezfsIqtUxDgqSSg8y, c: ViCDhQ2GOE, d: 20:38:42, }"] [2025/04/04 20:24:53.168 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:43, a: 889, b: 889, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:38:43, a: 889, b: 889, QH1PS: CCGgwPNG8GLxYKwaOmglZc968shLs, c: p02RGHJT5, }"] [2025/04/04 20:24:53.268 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:44, a: 890, b: 890, }"] [row2="{ a: 890, b: 890, QH1PS: YRqQjRn4prIfn8rBMFAZaDGCn56UxAWtodSN19mDsq, c: T, d: 20:38:44, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2025/04/04 20:24:53.368 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 891, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:45, a: 891, }"] [row2="{ QH1PS: pCZl8kZfPTQVoXWeY6Qqmq3yFrNy3UonXY7ZCG3n, c: ft, d: 20:38:45, a: 891, b: 891, }"] [2025/04/04 20:24:53.468 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 892, b: 892, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:46, }"] [row2="{ a: 892, b: 892, QH1PS: wv0qmisqG6YTNLgmQTdARHvVJAdBmLB6PfZPXe2g, c: g011PZ, d: 20:38:46, }"] Run finished, takes 632.850287ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1845.8, Avg(us): 1235, Min(us): 786, Max(us): 91001, 95th(us): 2000, 99th(us): 2000 [2025/04/04 20:24:53.474 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:24:53.475 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:24:53.475 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:24:53.476 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:24:53.476 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118702381826082] [resolvedTs=457118708987854852] [2025/04/04 20:24:53.479 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:24:53.479 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:24:53.479 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:24:53.480 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.810352ms] [2025/04/04 20:24:53.484 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:24:53.487 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:24:53.488 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:24:53.489 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:24:53.489 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:24:53.488 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:24:53.493 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:24:53.502 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:24:53.502 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/04 20:24:53.568 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:38:47, a: 893, b: 893, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 893, b: 893, QH1PS: 2L, c: n9L6hH, d: 20:38:47, }"] [2025/04/04 20:24:53.669 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 894, b: 894, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:48, }"] [row2="{ c: C, d: 20:38:48, a: 894, b: 894, QH1PS: I41FpwsR, }"] [2025/04/04 20:24:53.769 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 895, b: 895, QH1PS: FD6aGa35nJOTAqUfWSSBOLAJjjAA1iQrQG5, c: Y05rUMeqb, d: 20:38:49, }"] running rebalance_table pass test: rebalance table [2025/04/04 20:24:53.869 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 896, b: 896, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:50, }"] [row2="{ d: 20:38:50, a: 896, b: 896, QH1PS: FFG9jUS, c: RKc, }"] [2025/04/04 20:24:53.970 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:38:51, a: 897, b: 897, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 897, QH1PS: vOpZ74irHcCYINoidHwwqSKVlBqhnMacZEkWJJKAaQf, c: pZMNwXnBcA, d: 20:38:51, a: 897, }"] ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "recordcount"="1000" "scanproportion"="0" "operationcount"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "table"="a5" "readallfields"="true" "insertproportion"="0" "mysql.db"="multi_rocks" "mysql.port"="4000" "workload"="core" "dotransactions"="false" "threadcount"="2" ********************************************** [2025/04/04 20:24:54.070 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:52, a: 898, b: 898, }"] [row2="{ QH1PS: DhdBxD5HzTMvAb4Yq99iSJUWkS7PCQ06r5dj, c: 4U3EvG2n, d: 20:38:52, a: 898, b: 898, }"] [2025/04/04 20:24:54.170 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 899, b: 899, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:53, }"] [row2="{ a: 899, b: 899, QH1PS: rHWmqSFPwSfJD99Li1CDp5wFM, c: m, d: 20:38:53, }"] [2025/04/04 20:24:54.270 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:55, a: 900, b: 900, }"] [row2="{ d: 20:38:55, a: 900, b: 900, QH1PS: GAVofTacY9IhAwhYSoauZ8HfAR57yhzghypxIhtgvp, c: vavfV, }"] [2025/04/04 20:24:54.370 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 901, b: 901, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:56, }"] [row2="{ c: IdClxrbaju, d: 20:38:56, a: 901, b: 901, QH1PS: M1Q6hXaPy5a9xYe3gFXQ5RARdNCYMi8, }"] [2025/04/04 20:24:54.470 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 902, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:57, a: 902, }"] [row2="{ a: 902, b: 902, QH1PS: 0Ox6KeWdehj7ItjzWq20zs6ar1si3aDeUEnzQqbzt3Z, c: rA86duC, d: 20:38:57, }"] Run finished, takes 610.342314ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1692.7, Avg(us): 1111, Min(us): 777, Max(us): 75028, 95th(us): 2000, 99th(us): 2000 [2025/04/04 20:24:54.571 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 903, b: 903, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:58, }"] [row2="{ d: 20:38:58, a: 903, b: 903, QH1PS: jCHuKXMoDckNuN6xTtH, c: NafP, }"] [2025/04/04 20:24:54.671 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 904, b: 904, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:38:59, }"] [row2="{ a: 904, b: 904, QH1PS: bITFu7RZ8g3lSpXe0gSbguu5cFXMnjr90ph6EsmhBne9, c: iBwgdJ, d: 20:38:59, }"] [2025/04/04 20:24:54.771 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:00, a: 905, b: 905, }"] [row2="{ QH1PS: TS5fxEP8XeXYb1M4GLnSfcsL2KbuAPHemr1KnRQNeO, c: zF, d: 20:39:00, a: 905, b: 905, }"] + set +x + tso='457118710018867202 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118710018867202 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:24:54 CST 2025] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3405834060.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) running list_processor [2025/04/04 20:24:54.871 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 906, b: 906, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:01, }"] [row2="{ a: 906, b: 906, QH1PS: 4b8p8k1faXcORU, c: l5c1B6otE, d: 20:39:01, }"] [2025/04/04 20:24:54.971 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 907, b: 907, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:02, }"] [row2="{ QH1PS: mifYBqaaqo82, c: 056mSSjC, d: 20:39:02, a: 907, b: 907, }"] table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later pass test: list processors + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001730ab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118710265020419 MinTableBarrierTs:457118710265020419 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118710265020419, checkpointTs: 457118710265020419, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118710265020420, checkpointTs: 457118710265020419, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118710265020421, checkpointTs: 457118710265020419, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67734995-1504-4c1b-a7fe-33f5d83b540a {"id":"67734995-1504-4c1b-a7fe-33f5d83b540a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a0af 67734995-1504-4c1b-a7fe-33f5d83b540a /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118710265020419,"min-table-barrier-ts":457118710265020419,"admin-job-type":0} /tidb/cdc/default/default/task/position/67734995-1504-4c1b-a7fe-33f5d83b540a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001730ab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118710265020419 MinTableBarrierTs:457118710265020419 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118710265020419, checkpointTs: 457118710265020419, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118710265020420, checkpointTs: 457118710265020419, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118710265020421, checkpointTs: 457118710265020419, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67734995-1504-4c1b-a7fe-33f5d83b540a {"id":"67734995-1504-4c1b-a7fe-33f5d83b540a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a0af 67734995-1504-4c1b-a7fe-33f5d83b540a /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118710265+ grep -q 'failed to get info:' 020419,"min-table-barrier-ts":457118710265020419,"admin-job-type":0} /tidb/cdc/default/default/task/position/67734995-1504-4c1b-a7fe-33f5d83b540a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001730ab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118710265020419 MinTableBarrierTs:457118710265020419 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118710265020419, checkpointTs: 457118710265020419, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118710265020420, checkpointTs: 457118710265020419, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118710265020421, checkpointTs: 457118710265020419, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67734995-1504-4c1b-a7fe-33f5d83b540a {"id":"67734995-1504-4c1b-a7fe-33f5d83b540a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a0af 67734995-1504-4c1b-a7fe-33f5d83b540a /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118710265020419,"min-table-barrier-ts":457118710265020419,"admin-job-type":0} /tidb/cdc/default/default/task/position/67734995-1504-4c1b-a7fe-33f5d83b540a/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2025/04/04 20:24:55.071 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:03, a: 908, b: 908, }"] [row2="{ c: lRx1TFHadd, d: 20:39:03, a: 908, b: 908, QH1PS: 5890Np6m6RNrlsn1ivIX9hv0QqfASmDxfkc, }"] [2025/04/04 20:24:55.172 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:39:04, a: 909, b: 909, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:39:04, a: 909, b: 909, QH1PS: FCPtKvcDYl2TaBgMwMe6zgWX4IzVJDbYVzhjenAcSM, c: bTKAm9Osc, }"] [2025/04/04 20:24:55.272 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:39:06, a: 910, b: 910, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 910, b: 910, QH1PS: 3TN5OkpjNjxZXvaoKjktHhYvl7g7n1mwHy6j1bkttIOG, c: d, d: 20:39:06, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:24:55.508 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=2.032152902s] [2025/04/04 20:24:55.509 +08:00] [ERROR] [redo.go:272] ["ignore unsupported DDL"] [ddl="{\"StartTs\":457118702381826077,\"CommitTs\":457118702381826082,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file\",\"Table\":\"t1\",\"TableID\":118,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:24:55.372 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 911, b: 911, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:07, }"] [row2="{ a: 911, b: 911, QH1PS: dAw0UgfdarQZdW6O, c: r, d: 20:39:07, }"] [2025/04/04 20:24:55.472 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:39:08, a: 912, b: 912, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: m8uH, d: 20:39:08, a: 912, b: 912, QH1PS: fXbV2AlPHNSliLrech4o, }"] [2025/04/04 20:24:55.572 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:39:09, a: 913, b: 913, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 913, QH1PS: 1wJJTUhXhRT7oM3o5kWdWnXzlbQVZ, c: GqutdQJEq, d: 20:39:09, a: 913, }"] [2025/04/04 20:24:55.672 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 914, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:10, a: 914, }"] [row2="{ a: 914, b: 914, QH1PS: vQV0KggfDQdZFmoQr6R, c: gj16, d: 20:39:10, }"] [2025/04/04 20:24:55.773 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 915, b: 915, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:11, }"] [row2="{ a: 915, b: 915, QH1PS: belByGsIdf7G, c: 84Q, d: 20:39:11, }"] running get_processor [2025/04/04 20:24:55.873 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:12, a: 916, b: 916, }"] [row2="{ c: qwQyjFVqv, d: 20:39:12, a: 916, b: 916, QH1PS: X7zvuYZJyyqDSufbGXMgmwDpHhbtnmpg, }"] [2025/04/04 20:24:55.973 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 917, b: 917, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:13, }"] [row2="{ a: 917, b: 917, QH1PS: bSGl8Yg98F0blj, c: X9WdO, d: 20:39:13, }"] chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:24:56.073 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 918, b: 918, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:14, }"] [row2="{ a: 918, b: 918, QH1PS: R9K0BM97YHnCMZYRpzQvH3Dls6dPHs82LO5, c: WS, d: 20:39:14, }"] [2025/04/04 20:24:56.173 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:15, a: 919, b: 919, }"] [row2="{ QH1PS: c3iYN1ZvXkWGbc5mk, c: E9jesu4Y, d: 20:39:15, a: 919, b: 919, }"] [2025/04/04 20:24:56.274 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 920, b: 920, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:16, }"] [row2="{ b: 920, QH1PS: wsAuxh3GXlhWnjik7WeyfRvyixRINM, c: unHPSc5, d: 20:39:16, a: 920, }"] [2025/04/04 20:24:56.374 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 921, b: 921, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:17, }"] [row2="{ a: 921, b: 921, QH1PS: mHxrUgjb2FyPvRtxKSnQRwrdieFYo, c: psdYSYTSJW, d: 20:39:17, }"] [2025/04/04 20:24:56.474 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 922, b: 922, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:18, }"] [row2="{ c: kIOCHfdSKr, d: 20:39:18, a: 922, b: 922, QH1PS: UJz, }"] table multi_rocks.finish_mark not exists for 1-th check, retry later "is-owner": true, run task successfully [2025/04/04 20:24:56.574 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:39:19, a: 923, b: 923, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 923, b: 923, QH1PS: uSDeuMuJkVgPsLMx8JvA3E2J3w, c: QaYQClZI, d: 20:39:19, }"] [2025/04/04 20:24:56.674 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 924, b: 924, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:20, }"] [row2="{ c: XEHvo, d: 20:39:20, a: 924, b: 924, QH1PS: wzfQpBHrZxHfKGNB3aHIAnyibd1qOVD, }"] [2025/04/04 20:24:56.774 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:39:21, a: 925, b: 925, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:39:21, a: 925, b: 925, QH1PS: cXcGaa, c: 2, }"] table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later [2025/04/04 20:24:56.875 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 926, b: 926, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:22, }"] [row2="{ a: 926, b: 926, QH1PS: dM8LjgZNhrcMcMw8hKUEQ, c: jz5HfZ7, d: 20:39:22, }"] [2025/04/04 20:24:56.975 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 927, b: 927, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:23, }"] [row2="{ a: 927, b: 927, QH1PS: Uffkg2Rf0Trug5eZbmYJN6jCGOY, c: a, d: 20:39:23, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:57.075 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 928, b: 928, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:24, }"] [row2="{ d: 20:39:24, a: 928, b: 928, QH1PS: wi9KPqfQVjXFeCZvXZt2eNyoLeIdyRov5rSzbg, c: wHzl, }"] [2025/04/04 20:24:57.175 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:25, a: 929, b: 929, }"] [row2="{ QH1PS: Gw8QpSTjlogJKo3FQBDOrJfO9Pcmi9re8XEWPIlx, c: A3rD0Fzk, d: 20:39:25, a: 929, b: 929, }"] [2025/04/04 20:24:57.275 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 930, b: 930, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:26, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:24:57.157 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118708987854852] [2025/04/04 20:24:57.157 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118708987854852] [2025/04/04 20:24:57.257 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118708987854852] [2025/04/04 20:24:57.258 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118708987854852] [2025/04/04 20:24:57.375 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 931, b: 931, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:27, }"] [2025/04/04 20:24:57.476 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:28, a: 932, b: 932, }"] [2025/04/04 20:24:57.576 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:39:29, a: 933, b: 933, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:57.560 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457118708987854852] [2025/04/04 20:24:57.560 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457118708987854852] [2025/04/04 20:24:57.660 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=457118708987854852] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:24:57.660 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] Apply redo log successfully [2025/04/04 20:24:57.676 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 934, b: 934, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:30, }"] [2025/04/04 20:24:57.776 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:31, a: 935, b: 935, }"] owner pid: 6421 owner id 67734995-1504-4c1b-a7fe-33f5d83b540a process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '67734995-1504-4c1b-a7fe-33f5d83b540a' /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a0af run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:24:57 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1548779-410e-486b-b85f-cc361538c859 {"id":"d1548779-410e-486b-b85f-cc361538c859","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769494} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2b09ae9 d1548779-410e-486b-b85f-cc361538c859 /tidb/cdc/default/default/upstream/7489432901003976052 {"id":7489432901003976052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1548779-410e-486b-b85f-cc361538c859 {"id":"d1548779-410e-486b-b85f-cc361538c859","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769494} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2b09ae9 d1548779-410e-486b-b85f-cc361538c859 /tidb/cdc/default/default/upstream/7489432901003976052 {"id":7489432901003976052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1548779-410e-486b-b85f-cc361538c859 {"id":"d1548779-410e-486b-b85f-cc361538c859","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769494} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2b09ae9 d1548779-410e-486b-b85f-cc361538c859 /tidb/cdc/default/default/upstream/7489432901003976052 {"id":7489432901003976052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34114.out cli changefeed create --start-ts=457118710018867202 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2025/04/04 20:24:57.876 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 936, b: 936, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:32, }"] [2025/04/04 20:24:57.977 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 937, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:33, a: 937, }"] [2025/04/04 20:24:58.077 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 938, b: 938, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:34, }"] Create changefeed successfully! ID: fc815418-c1e8-4f0a-986a-201eaa9f72cf Info: {"upstream_id":7489432901003976052,"namespace":"default","id":"fc815418-c1e8-4f0a-986a-201eaa9f72cf","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:24:58.148878913+08:00","start_ts":457118710018867202,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118710018867202,"checkpoint_ts":457118710018867202,"checkpoint_time":"2025-04-04 20:24:53.175"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:24:58.177 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 939, b: 939, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:35, }"] [2025/04/04 20:24:58.277 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:39:37, a: 940, b: 940, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] table multi_rocks.finish_mark not exists for 2-th check, retry later pass test: get processor [2025/04/04 20:24:58.377 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:39:38, a: 941, b: 941, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:58.477 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:39, a: 942, b: 942, }"] [2025/04/04 20:24:58.578 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 943, b: 943, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:40, }"] check diff successfully [Fri Apr 4 20:24:58 CST 2025] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later [2025/04/04 20:24:58.678 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:41, a: 944, b: 944, }"] [2025/04/04 20:24:58.778 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:39:42, a: 945, b: 945, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:24:58.878 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 946, b: 946, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:43, }"] [2025/04/04 20:24:58.978 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 947, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:44, a: 947, }"] [2025/04/04 20:24:59.078 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:39:45, a: 948, b: 948, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432927293192520 TiDB Cluster ID. tikv_gc_leader_uuid 658030bb904000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:11927, start at 2025-04-04 20:24:57.180935796 +0800 CST m=+2.431710211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:57.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:57.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:57.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432927293192520 TiDB Cluster ID. tikv_gc_leader_uuid 658030bb904000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:11927, start at 2025-04-04 20:24:57.180935796 +0800 CST m=+2.431710211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:57.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:57.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:57.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432939089072939 TiDB Cluster ID. tikv_gc_leader_uuid 658030bbb740005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-l-jjkm6, pid:11995, start at 2025-04-04 20:24:57.316170699 +0800 CST m=+2.510098337 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:57.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:57.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:57.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:24:59.179 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 949, b: 949, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:46, }"] [2025/04/04 20:24:59.279 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 950, b: 950, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:48, }"] [row2="{ b: 950, QH1PS: H690B7H7yu1, c: ewFfkP, d: 20:39:48, a: 950, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release running move_table [2025/04/04 20:24:59.379 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 951, b: 951, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:49, }"] [row2="{ a: 951, b: 951, QH1PS: q, c: 1XqA9w2bl, d: 20:39:49, }"] [2025/04/04 20:24:59.479 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:39:50, a: 952, b: 952, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 952, QH1PS: 3CZE2aLdNdlw7fLCs3MSCHgAUdAK519koixxLuDi, c: vtqMUK, d: 20:39:50, a: 952, }"] [2025/04/04 20:24:59.579 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 953, b: 953, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:51, }"] [row2="{ a: 953, b: 953, QH1PS: DPThdItDpyzu6BsGPN, c: KkWJ077Pg, d: 20:39:51, }"] + set +x pass test: move table [2025/04/04 20:24:59.680 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 954, b: 954, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:52, }"] [row2="{ QH1PS: BKdov3OdH84LN0xCuWtZXccMLSl, c: 7t0, d: 20:39:52, a: 954, b: 954, }"] [2025/04/04 20:24:59.780 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 955, b: 955, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:53, }"] [row2="{ a: 955, b: 955, QH1PS: 2U0pRSJjVP1, c: VyEb7GB3k, d: 20:39:53, }"] nonempty select id, val from test.availability1 where id=2 and val=22 [2025/04/04 20:24:59.880 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:39:54, a: 956, b: 956, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ d: 20:39:54, a: 956, b: 956, QH1PS: Kb2lVI4quh8, c: ltX5po, }"] [2025/04/04 20:24:59.980 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 957, b: 957, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:55, }"] [row2="{ a: 957, b: 957, QH1PS: qy, c: o, d: 20:39:55, }"] [2025/04/04 20:25:00.080 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 958, b: 958, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:56, }"] [row2="{ a: 958, b: 958, QH1PS: oU6DOUBPSo54Dwn4TwbO7v6AHfXEh5, c: pR36nK, d: 20:39:56, }"] run task successfully table multi_rocks.finish_mark not exists for 3-th check, retry later [2025/04/04 20:25:00.180 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 959, b: 959, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:57, }"] [row2="{ c: kKfa20cfp, d: 20:39:57, a: 959, b: 959, QH1PS: SKFE38zhFb8onAb8vVhK, }"] [2025/04/04 20:25:00.281 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 960, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:39:58, a: 960, }"] [2025/04/04 20:25:00.381 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:39:59, a: 961, b: 961, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [2025/04/04 20:25:00.481 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 962, b: 962, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:00, }"] [2025/04/04 20:25:00.581 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 963, b: 963, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:01, }"] wait process cdc.test exit for 1-th time... table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later [2025/04/04 20:25:00.681 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:40:02, a: 964, b: 964, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:25:00.782 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 20:40:03, a: 965, b: 965, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] wait process cdc.test exit for 2-th time... table generate_column.t1 not exists for 1-th check, retry later running set_log_level pass test: set log level [2025/04/04 20:25:00.882 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:04, a: 966, b: 966, }"] [2025/04/04 20:25:00.982 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: GNndyDUZ, d: 20:40:05, a: 967, b: 967, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [2025/04/04 20:25:01.082 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 968, b: 968, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:06, }"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13228.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/04 20:25:01.182 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 969, b: 969, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:07, }"] [2025/04/04 20:25:01.282 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 970, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:08, a: 970, }"] [row2="{ b: 970, QH1PS: ajRTJiI4NXAmZQSIgIbSueDnL4jfF5FSQ3jRP02rPLIi, c: N4TBej, d: 20:40:08, a: 970, }"] [2025/04/04 20:25:01.383 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 971, b: 971, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:09, }"] [row2="{ d: 20:40:09, a: 971, b: 971, QH1PS: D5, c: t, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Fri Apr 4 20:25:01 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.66136615.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:25:01.483 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: GNndyDUZ, d: 20:40:10, a: 972, b: 972, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ a: 972, b: 972, QH1PS: es4DlLFWLHQV8DLYcOvhEGdectcK3OKTv, c: e, d: 20:40:10, }"] [2025/04/04 20:25:01.583 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 973, b: 973, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:11, }"] [row2="{ b: 973, QH1PS: dT2JvjfMtv58ydvOIYWcYidXhTGr3z, c: q9YPn4h8, d: 20:40:11, a: 973, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432946661611347 TiDB Cluster ID. tikv_gc_leader_uuid 658030bd9a80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:9988, start at 2025-04-04 20:24:59.269475579 +0800 CST m=+2.228878693 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:59.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:59.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:59.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432946661611347 TiDB Cluster ID. tikv_gc_leader_uuid 658030bd9a80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:9988, start at 2025-04-04 20:24:59.269475579 +0800 CST m=+2.228878693 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:59.280 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:59.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:59.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432945315292487 TiDB Cluster ID. tikv_gc_leader_uuid 658030bdb64000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-7-d7hgk, pid:10048, start at 2025-04-04 20:24:59.370599905 +0800 CST m=+2.269173260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:26:59.379 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:24:59.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:14:59.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:25:01.683 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 974, b: 974, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:12, }"] [row2="{ d: 20:40:12, a: 974, b: 974, QH1PS: 2L, c: IrV, }"] [2025/04/04 20:25:01.783 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 975, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:13, a: 975, }"] [row2="{ d: 20:40:13, a: 975, b: 975, QH1PS: n8UNgPhsgg, c: udGs87ly5, }"] [2025/04/04 20:25:01.884 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 976, b: 976, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:14, }"] [row2="{ d: 20:40:14, a: 976, b: 976, QH1PS: YIbY1JfGaFqIvboZwr0RgfSewwnS2Bi7baEz5lXYx, c: jvq, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) running remove_changefeed [2025/04/04 20:25:01.984 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 977, b: 977, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:15, }"] [row2="{ a: 977, b: 977, QH1PS: ckqClPBDj, c: RtO, d: 20:40:15, }"] [2025/04/04 20:25:02.084 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 978, b: 978, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:16, }"] [row2="{ a: 978, b: 978, QH1PS: zZEz7iECLLz8TqCv5o8tOkFl0aPwfGPuWU0froUAz6SiE, c: BPP, d: 20:40:16, }"] table multi_rocks.finish_mark exists [2025/04/04 20:25:02.184 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 979, b: 979, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:17, }"] [row2="{ b: 979, QH1PS: fQMmvQs5NyUgQQ6e8LI, c: 2UzQ, d: 20:40:17, a: 979, }"] [2025/04/04 20:25:02.284 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:18, a: 980, b: 980, }"] [row2="{ b: 980, QH1PS: v4Derxuo, c: cmn3SF, d: 20:40:18, a: 980, }"] [2025/04/04 20:25:02.384 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 981, b: 981, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:19, }"] [row2="{ d: 20:40:19, a: 981, b: 981, QH1PS: 76F5KBZUV6NpdkeDfw3NTDdhtP0Z3Z8OBTUt6TG8CYw, c: H7bp5RzE, }"] check diff successfully pass test: remove changefeed table changefeed_auto_stop_4.usertable exists [2025/04/04 20:25:02.485 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 982, b: 982, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:20, }"] [row2="{ a: 982, b: 982, QH1PS: GSqqhDeY9Zd4RK, c: oqH, d: 20:40:20, }"] [2025/04/04 20:25:02.585 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:40:21, a: 983, b: 983, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ QH1PS: 0f6exd4bgBgYFUF00G1M77eAoej9XDlD5q, c: o, d: 20:40:21, a: 983, b: 983, }"] check diff successfully [2025/04/04 20:25:02.685 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:40:22, a: 984, b: 984, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ b: 984, QH1PS: wNsq6uXJgPposyFR6bo794KmZNAeaosD5riqVnHiHWs7T, c: FP, d: 20:40:22, a: 984, }"] [2025/04/04 20:25:02.785 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 20:40:23, a: 985, b: 985, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ c: EM3397, d: 20:40:23, a: 985, b: 985, QH1PS: qUDqeUyMnptXte5aLSCjFidVPnTYjZ7jp5Slrgnr7, }"] [2025/04/04 20:25:02.885 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:40:24, a: 986, b: 986, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ b: 986, QH1PS: lApzwm5a89Tw0mPyMTtJGORBzn1A49shNgMcK, c: s, d: 20:40:24, a: 986, }"] table generate_column.t1 exists + set +x + tso='457118712188895234 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118712188895234 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x wait process cdc.test exit for 1-th time... [2025/04/04 20:25:02.986 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 987, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:25, a: 987, }"] [row2="{ QH1PS: zfO4MCH2F34aBtI1pHymnJ74h8jyT4mfnbT8z31zm, c: K, d: 20:40:25, a: 987, b: 987, }"] [2025/04/04 20:25:03.086 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ b: 988, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:26, a: 988, }"] [row2="{ a: 988, b: 988, QH1PS: aiufoGZpZTItI, c: 0YJ, d: 20:40:26, }"] [Fri Apr 4 20:25:03 CST 2025] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1327113273.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... [2025/04/04 20:25:03.186 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 989, b: 989, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:27, }"] [row2="{ QH1PS: yyvZEi52PS, c: M9, d: 20:40:27, a: 989, b: 989, }"] [2025/04/04 20:25:03.286 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:28, a: 990, b: 990, }"] [row2="{ d: 20:40:28, a: 990, b: 990, QH1PS: czfQntr2WHpaJtuhjvOd, c: kFHSPQqsqp, }"] [2025/04/04 20:25:03.386 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:29, a: 991, b: 991, }"] [row2="{ a: 991, b: 991, QH1PS: vkOYoqcdyskDgJFPEugoF6gQXYRWR7il7, c: I, d: 20:40:29, }"] table generate_column.t2 not exists for 1-th check, retry later running resign_owner [Fri Apr 4 20:25:03 CST 2025] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.1128611288.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2025/04/04 20:25:03.486 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:30, a: 992, b: 992, }"] [row2="{ d: 20:40:30, a: 992, b: 992, QH1PS: EsogWTUQ, c: h, }"] [2025/04/04 20:25:03.587 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ d: 20:40:31, a: 993, b: 993, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, }"] [row2="{ a: 993, b: 993, QH1PS: DwaKjgGLVyD1yINyz, c: Pzk, d: 20:40:31, }"] pass test: resign owner ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:25:03.687 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ c: GNndyDUZ, d: 20:40:32, a: 994, b: 994, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, }"] [row2="{ QH1PS: agsBtcGj1, c: MzTKRmE6K9, d: 20:40:32, a: 994, b: 994, }"] [2025/04/04 20:25:03.787 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 995, b: 995, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:33, }"] [row2="{ a: 995, b: 995, QH1PS: Mg, c: WDzMZn3P, d: 20:40:33, }"] [2025/04/04 20:25:03.888 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:34, a: 996, b: 996, }"] [row2="{ a: 996, b: 996, QH1PS: avEzDhNurCprUk, c: 3, d: 20:40:34, }"] wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:03 CST 2025] <<<<<< run test case multi_rocks success! >>>>>> [2025/04/04 20:25:03.988 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ a: 997, b: 997, QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:35, }"] [row2="{ a: 997, b: 997, QH1PS: A76UawvWB6zkT927NjKlxqWXReGSnjhGL9QTBhwLWCGrS, c: eylm2kY1, d: 20:40:35, }"] [2025/04/04 20:25:04.088 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:36, a: 998, b: 998, }"] [row2="{ a: 998, b: 998, QH1PS: cr7j, c: vUG2, d: 20:40:36, }"] [2025/04/04 20:25:04.188 +08:00] [WARN] [diff.go:937] ["find different row"] [column=QH1PS] [row1="{ QH1PS: ZeZAHtiSR6XGRXNdMNfjQdKY, c: GNndyDUZ, d: 20:40:37, a: 999, b: 999, }"] [row2="{ c: 4HuJ8Xvz0, d: 20:40:37, a: 999, b: 999, QH1PS: Y, }"] [2025/04/04 20:25:04.288 +08:00] [WARN] [diff.go:766] ["rows is not equal"] [table=`test`.`itest`] [where="((TRUE) AND TRUE)"] [cost=1m30.447178482s] [2025/04/04 20:25:04.291 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2025/04/04 20:24:55.890 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2025/04/04 20:24:56.130 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:24:56.130 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:24:56.333 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:24:56.335 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:24:56.340 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:24:56.341 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:24:57.551 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:24:57.568 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:24:57.762 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:24:57.764 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:24:57.802 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:24:57.806 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:24:57.975 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:24:58.014 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:24:59.207 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:24:59.210 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:24:59.250 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:24:59.258 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:24:59.447 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:24:59.489 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:24:59.680 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:24:59.684 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:24:59.722 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:24:59.729 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:04 CST 2025] <<<<<< run test case changefeed_auto_stop success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014ba3f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118711575740419 MinTableBarrierTs:457118712886460421 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b565f78-d027-4639-8b1e-bc5b8afd2f2c {"id":"0b565f78-d027-4639-8b1e-bc5b8afd2f2c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769501} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a11d 0b565f78-d027-4639-8b1e-bc5b8afd2f2c /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118711575740419,"min-table-barrier-ts":457118712886460421,"admin-job-type":0} /tidb/cdc/default/default/task/position/0b565f78-d027-4639-8b1e-bc5b8afd2f2c/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014ba3f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118711575740419 MinTableBarrierTs:457118712886460421 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b565f78-d027-4639-8b1e-bc5b8afd2f2c {"id":"0b565f78-d027-4639-8b1e-bc5b8afd2f2c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769501} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a11d 0b565f78-d027-4639-8b1e-bc5b8afd2f2c /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118711575740419,"min-table-barrier-ts":457118712886460421,"admin-job-type":0} /tidb/cdc/default/default/task/position/0b565f78-d027-4639-8b1e-bc5b8afd2f2c/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/330a1306-950b-43d9-93e3-10dbf8731a51 {UpstreamID:7489431843147851979 Namespace:default ID:330a1306-950b-43d9-93e3-10dbf8731a51 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:20:57.770561931 +0800 CST StartTs:457118646419849217 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014ba3f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118648307286017} {CheckpointTs:457118711575740419 MinTableBarrierTs:457118712886460421 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118711575740419, checkpointTs: 457118711575740419, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b565f78-d027-4639-8b1e-bc5b8afd2f2c {"id":"0b565f78-d027-4639-8b1e-bc5b8afd2f2c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769501} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319600bee8a11d 0b565f78-d027-4639-8b1e-bc5b8afd2f2c /tidb/cdc/default/default/changefeed/info/330a1306-950b-43d9-93e3-10dbf8731a51 {"upstream-id":7489431843147851979,"namespace":"default","changefeed-id":"330a1306-950b-43d9-93e3-10dbf8731a51","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:20:57.770561931+08:00","start-ts":457118646419849217,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118648307286017} /tidb/cdc/default/default/changefeed/status/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":457118711575740419,"min-table-barrier-ts":457118712886460421,"admin-job-type":0} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/0b565f78-d027-4639-8b1e-bc5b8afd2f2c/330a1306-950b-43d9-93e3-10dbf8731a51 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489431843147851979 {"id":7489431843147851979,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' running get_tso pass test: get tso table generate_column.t2 exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432968037851073 TiDB Cluster ID. tikv_gc_leader_uuid 658030c21fc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:12739, start at 2025-04-04 20:25:03.914593396 +0800 CST m=+2.542312349 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:03.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:03.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:03.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432968037851073 TiDB Cluster ID. tikv_gc_leader_uuid 658030c21fc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:12739, start at 2025-04-04 20:25:03.914593396 +0800 CST m=+2.542312349 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:03.961 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:03.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:03.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489432960972727029 TiDB Cluster ID. tikv_gc_leader_uuid 658030c238c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:12818, start at 2025-04-04 20:25:04.005580426 +0800 CST m=+2.544028105 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:04.013 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:04.021 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:04.021 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } running verify_table "is-owner": true, run task successfully pass test: verify table + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:06 GMT < Content-Length: 812 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/ffd2341c-fdc7-4643-a00a-1e2473544b10 {"id":"ffd2341c-fdc7-4643-a00a-1e2473544b10","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769503} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22319600c2cec8fb ffd2341c-fdc7-4643-a00a-1e2473544b10 /tidb/cdc/test1/default/upstream/7489432927293192520 {"id":7489432927293192520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/ffd2341c-fdc7-4643-a00a-1e2473544b10 {"id":"ffd2341c-fdc7-4643-a00a-1e2473544b10","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769503} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22319600c2cec8fb ffd2341c-fdc7-4643-a00a-1e2473544b10 /tidb/cdc/test1/default/upstream/7489432927293192520 {"id":7489432927293192520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/ffd2341c-fdc7-4643-a00a-1e2473544b10 {"id":"ffd2341c-fdc7-4643-a00a-1e2473544b10","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769503} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22319600c2cec8fb ffd2341c-fdc7-4643-a00a-1e2473544b10 /tidb/cdc/test1/default/upstream/7489432927293192520 {"id":7489432927293192520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 20:25:06 CST 2025] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1332513327.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:06 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e6d7324-53a0-4368-ac1d-fdba203c8289 {"id":"7e6d7324-53a0-4368-ac1d-fdba203c8289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769503} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2d9dfd5 7e6d7324-53a0-4368-ac1d-fdba203c8289 /tidb/cdc/default/default/upstream/7489432946661611347 {"id":7489432946661611347,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e6d7324-53a0-4368-ac1d-fdba203c8289 {"id":"7e6d7324-53a0-4368-ac1d-fdba203c8289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769503} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2d9dfd5 7e6d7324-53a0-4368-ac1d-fdba203c8289 /tidb/cdc/default/default/upstream/7489432946661611347 {"id":7489432946661611347,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e6d7324-53a0-4368-ac1d-fdba203c8289 {"id":"7e6d7324-53a0-4368-ac1d-fdba203c8289","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769503} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2d9dfd5 7e6d7324-53a0-4368-ac1d-fdba203c8289 /tidb/cdc/default/default/upstream/7489432946661611347 {"id":7489432946661611347,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2025/04/04 20:25:06.403 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2025/04/04 20:25:06.652 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:25:06.654 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:25:06.863 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:25:06.864 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:25:06.877 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:25:06.879 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] running create_changefeed_v2 [2025/04/04 20:25:07.064 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:25:07.092 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] owner pid: 6618 owner id 0b565f78-d027-4639-8b1e-bc5b8afd2f2c [2025/04/04 20:25:07.251 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:25:07.252 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:25:07.292 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:25:07.299 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:25:07.444 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:25:07.493 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:25:07.640 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:25:07.642 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:25:07.695 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:25:07.701 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] pass test: create changefeed v2 check diff successfully + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14133.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/04 20:25:07.843 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:25:07.901 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] wait process cdc.test exit for 1-th time... [2025/04/04 20:25:08.045 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:25:08.046 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:25:08.119 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:25:08.125 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:25:08.249 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:25:08.333 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:25:08.442 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:25:08.445 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... [2025/04/04 20:25:08.539 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:25:08.545 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:25:08.633 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:25:08.744 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/04 20:25:08.826 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:25:08.833 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:25:08.962 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:25:08.973 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] running delete_changefeed_v2 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:08 CST 2025] <<<<<< run test case generate_column success! >>>>>> [2025/04/04 20:25:09.027 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:25:09.162 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:25:09.213 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:25:09.220 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] table owner_remove_table_error.finished_mark not exists for 1-th check, retry later changefeed id: 330a1306-950b-43d9-93e3-10dbf8731a51 curl -s -d "cf-id=330a1306-950b-43d9-93e3-10dbf8731a51&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 489 755 --:--:-- --:--:-- --:--:-- 760 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/04 20:25:09.365 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:25:09.380 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/04 20:25:09.403 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] pass test: remove changefeed v2 + set +x + tso='457118713936871425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118713936871425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118713936871425 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 20:25:09 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1416814170.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:25:09.567 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:25:09.593 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:25:09.599 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:25:09.777 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:09 GMT < Content-Length: 812 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73 {"id":"658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769506} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22319600c2cec923 658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73 /tidb/cdc/test2/default/upstream/7489432927293192520 {"id":7489432927293192520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73 {"id":"658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769506} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22319600c2cec923 658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73 /tidb/cdc/test2/default/upstream/7489432927293192520 {"id":7489432927293192520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73 {"id":"658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769506} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22319600c2cec923 658d6cd6-2c0a-45fd-a639-ddbbd4ed3b73 /tidb/cdc/test2/default/upstream/7489432927293192520 {"id":7489432927293192520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13388.out cli changefeed create --start-ts=457118712188895234 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: 73451384-2f31-4274-8358-cbd2aec7c38c Info: {"upstream_id":7489432927293192520,"namespace":"default","id":"73451384-2f31-4274-8358-cbd2aec7c38c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:25:09.65929655+08:00","start_ts":457118712188895234,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118712188895234,"checkpoint_ts":457118712188895234,"checkpoint_time":"2025-04-04 20:25:01.453"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:25:09.795 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:25:09.859 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:25:10.046 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:25:10.066 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:25:10.073 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:25:10.228 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:25:10.240 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:25:10.256 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:25:10.419 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/04 20:25:10.463 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:25:10.470 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] running unsafe_apis [2025/04/04 20:25:10.600 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:25:10.613 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/04 20:25:10.660 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] [2025/04/04 20:25:10.787 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock [2025/04/04 20:25:09.922 +08:00] [WARN] [diff.go:557] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [sourceChecksum=0] [targetChecksum=3836333274] [getSourceChecksumCost=3.343514ms] [getTargetChecksumCost=10.394414ms] [2025/04/04 20:25:10.850 +08:00] [INFO] [main.go:221] ["0 insert success: 2200"] [2025/04/04 20:25:10.856 +08:00] [INFO] [main.go:235] ["0 delete success: 1100"] [2025/04/04 20:25:10.974 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] [2025/04/04 20:25:10.987 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] [2025/04/04 20:25:11.044 +08:00] [INFO] [main.go:221] ["0 insert success: 2300"] nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2025/04/04 20:25:11.160 +08:00] [INFO] [main.go:221] ["1 insert success: 2300"] [2025/04/04 20:25:11.242 +08:00] [INFO] [main.go:221] ["0 insert success: 2400"] [2025/04/04 20:25:11.252 +08:00] [INFO] [main.go:235] ["0 delete success: 1200"] [2025/04/04 20:25:11.354 +08:00] [INFO] [main.go:221] ["1 insert success: 2400"] [2025/04/04 20:25:11.366 +08:00] [INFO] [main.go:235] ["1 delete success: 1200"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.13432.out cli changefeed create --start-ts=457118712188895234 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml Create changefeed successfully! ID: 50dd1caf-8b21-43e6-99fb-e1e6214aea44 Info: {"upstream_id":7489432927293192520,"namespace":"default","id":"50dd1caf-8b21-43e6-99fb-e1e6214aea44","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:25:11.618999046+08:00","start_ts":457118712188895234,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118712188895234,"checkpoint_ts":457118712188895234,"checkpoint_time":"2025-04-04 20:25:01.453"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table owner_remove_table_error.finished_mark not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:12 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c738f720-1142-4977-b7cb-568984658995 {"id":"c738f720-1142-4977-b7cb-568984658995","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2e8d3e5 c738f720-1142-4977-b7cb-568984658995 /tidb/cdc/default/default/upstream/7489432968037851073 {"id":7489432968037851073,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c738f720-1142-4977-b7cb-568984658995 {"id":"c738f720-1142-4977-b7cb-568984658995","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2e8d3e5 c738f720-1142-4977-b7cb-568984658995 /tidb/cdc/default/default/upstream/7489432968037851073 {"id":7489432968037851073,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c738f720-1142-4977-b7cb-568984658995 {"id":"c738f720-1142-4977-b7cb-568984658995","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c2e8d3e5 c738f720-1142-4977-b7cb-568984658995 /tidb/cdc/default/default/upstream/7489432968037851073 {"id":7489432968037851073,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118713936871425 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14225.out cli changefeed create --start-ts=457118713936871425 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + set +x Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489432968037851073,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:25:13.06312303+08:00","start_ts":457118713936871425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118713936871425,"checkpoint_ts":457118713936871425,"checkpoint_time":"2025-04-04 20:25:08.121"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.multi_cdc1 exists table test.multi_cdc2 exists wait process cdc.test exit for 3-th time... check diff failed 1-th time, retry later table owner_remove_table_error.finished_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:25:13 CST 2025] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ci_collation_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 2-th time... Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:14 CST 2025] <<<<<< run test case owner_remove_table_error success! >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass [Pipeline] // podTemplate check diff successfully [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:16 CST 2025] <<<<<< run test case availability success! >>>>>> table test.t1 exists + sleep 5 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:17 CST 2025] <<<<<< run test case multi_cdc_cluster success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/ci_collation_compatibility Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/04 20:25:17.290 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2025/04/04 20:25:17.499 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:25:17.505 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:25:17.683 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:25:17.685 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:25:17.698 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:25:17.699 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:25:17.875 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:25:17.894 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:25:18.062 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:25:18.064 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:25:18.081 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:25:18.082 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:25:18.385 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:25:18.479 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:25:18.679 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:25:18.681 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:25:18.702 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:25:18.703 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:25:18.867 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:25:18.888 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [2025/04/04 20:25:19.064 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:25:19.066 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:25:19.086 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:25:19.087 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [2025/04/04 20:25:19.260 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:25:19.288 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2025/04/04 20:25:19.456 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:25:19.458 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:25:19.485 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:25:19.486 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:25:19.649 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/04 20:25:19.669 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [2025/04/04 20:25:19.840 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:25:19.842 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:25:19.856 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:25:19.857 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [Pipeline] } [2025/04/04 20:25:20.029 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:25:20.041 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:25:20.270 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:25:20.272 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:25:20.276 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:25:20.279 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:25:20.625 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:25:20.626 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:25:20.816 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:25:20.817 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:25:20.821 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:25:20.823 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433030786483696 TiDB Cluster ID. tikv_gc_leader_uuid 658030d0590000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:12925, start at 2025-04-04 20:25:18.478121555 +0800 CST m=+2.143462550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:18.486 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:18.487 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:18.487 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433030786483696 TiDB Cluster ID. tikv_gc_leader_uuid 658030d0590000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:12925, start at 2025-04-04 20:25:18.478121555 +0800 CST m=+2.143462550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:18.486 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:18.487 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:18.487 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433031568857389 TiDB Cluster ID. tikv_gc_leader_uuid 658030d096c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:13011, start at 2025-04-04 20:25:18.691337208 +0800 CST m=+2.245162014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:18.683 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:18.683 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:25:21.018 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:25:21.021 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:25:21.204 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:25:21.206 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:25:21.206 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:25:21.207 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:25:21.386 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/04 20:25:21.396 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:25:21.585 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:25:21.587 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/04 20:25:21.590 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/04 20:25:21.592 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:25:21.788 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] [2025/04/04 20:25:21.789 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] [2025/04/04 20:25:21.973 +08:00] [INFO] [main.go:235] ["0 delete success: 1100"] [2025/04/04 20:25:21.974 +08:00] [INFO] [main.go:221] ["0 insert success: 2200"] [2025/04/04 20:25:21.978 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] [2025/04/04 20:25:21.980 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] [2025/04/04 20:25:22.163 +08:00] [INFO] [main.go:221] ["0 insert success: 2300"] [2025/04/04 20:25:22.169 +08:00] [INFO] [main.go:221] ["1 insert success: 2300"] + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12143 36.6 0.6 4894060 2542092 ? Sl 20:24 0:08 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12144 25.5 0.6 4844396 2488456 ? Sl 20:24 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12145 25.9 0.6 4848496 2476108 ? Sl 20:24 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12147 35.4 0.6 4871532 2524948 ? Sl 20:24 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3720 0 --:--:-- --:--:-- --:--:-- 3738 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:25:20.422","puller_resolved_ts":"2025-04-04 20:25:14.521","last_synced_ts":"2025-04-04 20:25:14.571","now_ts":"2025-04-04 20:25:21.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:25:20.422","puller_resolved_ts":"2025-04-04' '20:25:14.521","last_synced_ts":"2025-04-04' '20:25:14.571","now_ts":"2025-04-04' '20:25:21.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:25:20.422","puller_resolved_ts":"2025-04-04' '20:25:14.521","last_synced_ts":"2025-04-04' '20:25:14.571","now_ts":"2025-04-04' '20:25:21.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [Fri Apr 4 20:25:22 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1434714349.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:25 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3199f54-4ade-4546-a948-5e59f9a76c0e {"id":"f3199f54-4ade-4546-a948-5e59f9a76c0e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c32732d7 f3199f54-4ade-4546-a948-5e59f9a76c0e /tidb/cdc/default/default/upstream/7489433030786483696 {"id":7489433030786483696,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3199f54-4ade-4546-a948-5e59f9a76c0e {"id":"f3199f54-4ade-4546-a948-5e59f9a76c0e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c32732d7 f3199f54-4ade-4546-a948-5e59f9a76c0e /tidb/cdc/default/default/upstream/7489433030786483696 {"id":7489433030786483696,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3199f54-4ade-4546-a948-5e59f9a76c0e {"id":"f3199f54-4ade-4546-a948-5e59f9a76c0e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769523} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c32732d7 f3199f54-4ade-4546-a948-5e59f9a76c0e /tidb/cdc/default/default/upstream/7489433030786483696 {"id":7489433030786483696,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "operationcount"="0" "scanproportion"="0" "mysql.db"="consistent_replicate_storage_file_large_value" "fieldlengthdistribution"="constant" "fieldcount"="1000" "insertproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "fieldlength"="15000" "threadcount"="10" "dotransactions"="false" "mysql.user"="root" "mysql.port"="4000" "readproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "recordcount"="100" "readallfields"="true" ********************************************** VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433057484337862 TiDB Cluster ID. tikv_gc_leader_uuid 658030d647c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:27748, start at 2025-04-04 20:25:24.512549784 +0800 CST m=+2.005426459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:24.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:24.511 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:24.511 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433057484337862 TiDB Cluster ID. tikv_gc_leader_uuid 658030d647c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:27748, start at 2025-04-04 20:25:24.512549784 +0800 CST m=+2.005426459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:24.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:24.511 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:24.511 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433050755790660 TiDB Cluster ID. tikv_gc_leader_uuid 658030d677c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-m8f71, pid:27830, start at 2025-04-04 20:25:24.746903158 +0800 CST m=+2.139819255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:24.754 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:24.753 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:24.753 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + SINK_TYPE=mysql + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.90.94 + export UP_TIDB_HOST=10.233.90.94 UP_PD_HOST_1=10.233.90.94 UP_PD_HOST_2=10.233.90.94 UP_PD_HOST_3=10.233.90.94 UP_TIKV_HOST_1=10.233.90.94 UP_TIKV_HOST_2=10.233.90.94 UP_TIKV_HOST_3=10.233.90.94 + UP_TIDB_HOST=10.233.90.94 + UP_PD_HOST_1=10.233.90.94 + UP_PD_HOST_2=10.233.90.94 + UP_PD_HOST_3=10.233.90.94 + UP_TIKV_HOST_1=10.233.90.94 + UP_TIKV_HOST_2=10.233.90.94 + UP_TIKV_HOST_3=10.233.90.94 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=13602 + trap 'stop_tidb_cluster && stop_proxy' EXIT + '[' mysql == mysql ']' + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.90.94' UP_TIDB_HOST 10.233.90.94 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.90.94' UP_PD_HOST_1 10.233.90.94 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.90.94' UP_PD_HOST_2 10.233.90.94 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.90.94' UP_PD_HOST_3 10.233.90.94 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.90.94' UP_TIKV_HOST_1 10.233.90.94 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.90.94' UP_TIKV_HOST_2 10.233.90.94 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.90.94' UP_TIKV_HOST_3 10.233.90.94 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2025/04/04 20:25:28.224 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2025/04/04 20:25:28.443 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:25:28.445 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:25:28.632 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:25:28.633 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:25:28.635 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:25:28.637 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.cli.29162.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/04 20:25:28.825 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:25:28.826 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:25:29.014 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:25:29.016 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:25:29.016 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:25:29.017 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:25:29.194 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:25:29.208 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:25:29.383 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:25:29.385 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:25:29.399 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:25:29.400 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:25:29.564 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:25:29.580 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:25:29.748 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:25:29.749 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:25:29.764 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:25:29.765 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:25:29.925 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:25:29.941 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... [2025/04/04 20:25:30.108 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:25:30.110 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:25:30.132 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:25:30.133 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:25:30.286 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/04 20:25:30.319 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:25:30.500 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:25:30.502 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:25:30.531 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:25:30.532 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] + set +x + tso='457118719465226241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118719465226241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:25:30 CST 2025] <<<<<< START cdc server in ci_collation_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.2919029192.out server --log-file /tmp/tidb_cdc_test/ci_collation_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ci_collation_compatibility/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:25:30.693 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:25:30.725 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:25:30.871 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:25:30.873 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/04 20:25:30.913 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:25:30.914 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:25:31.047 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:25:31.096 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:25:31.225 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:25:31.227 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:25:31.284 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:25:31.285 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:25:31.416 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:25:31.475 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:25:31.599 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:25:31.601 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:25:31.653 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:25:31.654 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:25:31.786 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/04 20:25:31.843 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:25:31.974 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:25:31.976 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/04 20:25:32.026 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/04 20:25:32.027 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:25:32.151 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] [2025/04/04 20:25:32.204 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] [2025/04/04 20:25:32.389 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] [2025/04/04 20:25:32.392 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] [2025/04/04 20:25:32.593 +08:00] [INFO] [main.go:235] ["0 delete success: 1100"] [2025/04/04 20:25:32.594 +08:00] [INFO] [main.go:221] ["0 insert success: 2200"] [2025/04/04 20:25:32.764 +08:00] [INFO] [main.go:221] ["1 insert success: 2300"] [2025/04/04 20:25:32.806 +08:00] [INFO] [main.go:221] ["0 insert success: 2300"] ***************** properties ***************** "insertproportion"="0" "recordcount"="10" "scanproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.db"="testSync" "dotransactions"="false" "mysql.port"="4000" "mysql.user"="root" "updateproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "threadcount"="2" "workload"="core" "readproportion"="0" ********************************************** Run finished, takes 6.411615ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2469.3, Avg(us): 1080, Min(us): 478, Max(us): 3658, 95th(us): 4000, 99th(us): 4000 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:25:32.953 +08:00] [INFO] [main.go:221] ["1 insert success: 2400"] [2025/04/04 20:25:32.956 +08:00] [INFO] [main.go:235] ["1 delete success: 1200"] [2025/04/04 20:25:32.990 +08:00] [INFO] [main.go:235] ["0 delete success: 1200"] [2025/04/04 20:25:32.991 +08:00] [INFO] [main.go:221] ["0 insert success: 2400"] start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:33 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ddca036-ceaa-44c6-b079-5ec8131b26d8 {"id":"0ddca036-ceaa-44c6-b079-5ec8131b26d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769530} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c33b69e4 0ddca036-ceaa-44c6-b079-5ec8131b26d8 /tidb/cdc/default/default/upstream/7489433057484337862 {"id":7489433057484337862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ddca036-ceaa-44c6-b079-5ec8131b26d8 {"id":"0ddca036-ceaa-44c6-b079-5ec8131b26d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769530} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c33b69e4 0ddca036-ceaa-44c6-b079-5ec8131b26d8 /tidb/cdc/default/default/upstream/7489433057484337862 {"id":7489433057484337862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ddca036-ceaa-44c6-b079-5ec8131b26d8 {"id":"0ddca036-ceaa-44c6-b079-5ec8131b26d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769530} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c33b69e4 0ddca036-ceaa-44c6-b079-5ec8131b26d8 /tidb/cdc/default/default/upstream/7489433057484337862 {"id":7489433057484337862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.cli.29251.out cli changefeed create --start-ts=457118719465226241 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?safe-mode=true' Create changefeed successfully! ID: 253fcb36-fab6-4991-b0ba-96912ab6881b Info: {"upstream_id":7489433057484337862,"namespace":"default","id":"253fcb36-fab6-4991-b0ba-96912ab6881b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T20:25:34.068546487+08:00","start_ts":457118719465226241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118719465226241,"checkpoint_ts":457118719465226241,"checkpoint_time":"2025-04-04 20:25:29.210"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Run finished, takes 8.991006942s INSERT_ERROR - Takes(s): 8.0, Count: 100, OPS: 12.4, Avg(us): 270444, Min(us): 110546, Max(us): 406358, 95th(us): 390000, 99th(us): 397000 table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433102428128958 TiDB Cluster ID. tikv_gc_leader_uuid 658030e04200012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:38498, start at 2025-04-04 20:25:34.749513004 +0800 CST m=+2.245110019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:34.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:34.728 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:34.728 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433102428128958 TiDB Cluster ID. tikv_gc_leader_uuid 658030e04200012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:38498, start at 2025-04-04 20:25:34.749513004 +0800 CST m=+2.245110019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:34.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:34.728 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:34.728 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433094373775198 TiDB Cluster ID. tikv_gc_leader_uuid 658030e0740000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:38571, start at 2025-04-04 20:25:34.951418047 +0800 CST m=+2.381335680 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:34.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:34.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:34.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_file_large_value.check1 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.90.94' (111) The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 [2025/04/04 20:25:39.111 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2025/04/04 20:25:39.313 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:25:39.316 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:25:39.501 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:25:39.503 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:25:39.510 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:25:39.511 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:25:39.701 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:25:39.711 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:25:39.885 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:25:39.886 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:25:39.902 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:25:39.904 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later [2025/04/04 20:25:40.071 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:25:40.085 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:25:40.258 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:25:40.260 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:25:40.276 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:25:40.278 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:25:40.442 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:25:40.461 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:25:40.632 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:25:40.633 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:25:40.643 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:25:40.645 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:25:40.820 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:25:40.832 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:25:41.012 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:25:41.012 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:25:41.013 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:25:41.014 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:25:41.196 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:25:41.197 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.90.94' (111) [2025/04/04 20:25:41.497 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:25:41.498 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:25:41.499 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:25:41.500 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:25:41.719 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:25:41.735 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/04 20:25:41.905 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:25:41.907 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:25:41.977 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:25:41.979 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later [2025/04/04 20:25:42.259 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:25:42.300 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:25:42.454 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:25:42.456 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:25:42.501 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:25:42.503 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:25:42.697 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:25:42.875 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:25:43.033 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:25:43.035 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:25:43.098 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:25:43.100 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:25:43.216 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:25:43.283 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433126539382157 TiDB Cluster ID. tikv_gc_leader_uuid 658030e6518000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:7909, start at 2025-04-04 20:25:40.976645267 +0800 CST m=+2.190463661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:40.984 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:40.984 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:40.984 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433126539382157 TiDB Cluster ID. tikv_gc_leader_uuid 658030e6518000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:7909, start at 2025-04-04 20:25:40.976645267 +0800 CST m=+2.190463661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:40.984 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:40.984 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:40.984 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433123336380534 TiDB Cluster ID. tikv_gc_leader_uuid 658030e677c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:7980, start at 2025-04-04 20:25:41.129370902 +0800 CST m=+2.278786235 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:41.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:41.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:41.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["10.233.90.94:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:25:43.403 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:25:43.405 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/04 20:25:43.585 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:25:43.587 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/04 20:25:43.736 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] [2025/04/04 20:25:43.798 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] Starting TLS TiDB... table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later [2025/04/04 20:25:43.931 +08:00] [INFO] [main.go:221] ["0 insert success: 2200"] [2025/04/04 20:25:43.933 +08:00] [INFO] [main.go:235] ["0 delete success: 1100"] [2025/04/04 20:25:43.975 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] [2025/04/04 20:25:43.977 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=9225 + echo 'proxy port: 13602' proxy port: 13602 + echo 'proxy pid: 9225' proxy pid: 9225 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=13602 go: downloading go1.23.4 (linux/amd64) table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433147551337604 TiDB Cluster ID. tikv_gc_leader_uuid 658030ea9f0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:40165, start at 2025-04-04 20:25:45.367715913 +0800 CST m=+1.476111392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:27:45.375 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:25:45.340 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:15:45.340 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40226.out cli tso query --pd=https://127.0.0.1:2579 table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later + set +x + tso='457118723955752987 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118723955752987 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:25:47 CST 2025] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.4027940281.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later + export http_proxy=http://127.0.0.1:13602 + http_proxy=http://127.0.0.1:13602 + export https_proxy=http://127.0.0.1:13602 + https_proxy=http://127.0.0.1:13602 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.90.94:2379' try to connect pd cluster via proxy, pd addr: 10.233.90.94:2379 + ensure 10 curl http://10.233.90.94:2379/ curl http://10.233.90.94:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13602; Connection refused run task failed 1-th time, retry later [2025/04/04 20:25:50.006 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2025/04/04 20:25:50.213 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:25:50.228 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:25:50.400 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:25:50.402 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:25:50.435 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:25:50.438 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:25:50.583 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] go: downloading google.golang.org/grpc v1.64.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/sys v0.28.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f [2025/04/04 20:25:50.625 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:25:50.759 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:25:50.761 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:25:50.813 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:25:50.815 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] go: downloading golang.org/x/text v0.21.0 go: downloading google.golang.org/protobuf v1.35.2 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2025/04/04 20:25:50.946 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:25:51.001 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:25:51.139 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:25:51.141 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:25:51.191 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:25:51.194 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:51 GMT < Content-Length: 1253 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f94edf29-c2c2-4ca9-ad3b-26946ccd410c {"id":"f94edf29-c2c2-4ca9-ad3b-26946ccd410c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769548} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c39204f1 f94edf29-c2c2-4ca9-ad3b-26946ccd410c /tidb/cdc/default/default/upstream/7489433147551337604 {"id":7489433147551337604,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f94edf29-c2c2-4ca9-ad3b-26946ccd410c {"id":"f94edf29-c2c2-4ca9-ad3b-26946ccd410c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769548} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c39204f1 f94edf29-c2c2-4ca9-ad3b-26946ccd410c /tidb/cdc/default/default/upstream/7489433147551337604 {"id":7489433147551337604,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f94edf29-c2c2-4ca9-ad3b-26946ccd410c {"id":"f94edf29-c2c2-4ca9-ad3b-26946ccd410c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769548} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d9600c39204f1 f94edf29-c2c2-4ca9-ad3b-26946ccd410c /tidb/cdc/default/default/upstream/7489433147551337604 {"id":7489433147551337604,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [2025/04/04 20:25:51.565 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:25:51.709 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] table consistent_replicate_storage_file_large_value.t2 exists check diff successfully [2025/04/04 20:25:51.893 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:25:51.896 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:25:51.967 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:25:51.970 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] curl http://10.233.90.94:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13602; Connection refused run task failed 2-th time, retry later [2025/04/04 20:25:52.311 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] wait process cdc.test exit for 1-th time... [2025/04/04 20:25:52.403 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:25:52.900 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] wait process cdc.test exit for 2-th time... [2025/04/04 20:25:52.903 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:25:52.958 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:25:52.962 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40342.out cli changefeed create --start-ts=457118723955752987 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [2025/04/04 20:25:53.259 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/04 20:25:53.563 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:25:53 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1463514637.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489433147551337604,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:25:53.631986149+08:00","start_ts":457118723955752987,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118723955752987,"checkpoint_ts":457118723955752987,"checkpoint_time":"2025-04-04 20:25:46.340"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:25:53.695 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:25:53.697 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:25:53.917 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:25:53.926 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:25:53.929 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:25:54.266 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:25:54.268 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/04 20:25:54.480 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:25:54.641 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] + set +x table test.simple not exists for 1-th check, retry later [2025/04/04 20:25:54.961 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:25:54.966 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] curl http://10.233.90.94:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13602; Connection refused run task failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:25:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/6fc6677d-b916-4c30-8c4b-61e64559c562 {UpstreamID:7489433030786483696 Namespace:default ID:6fc6677d-b916-4c30-8c4b-61e64559c562 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:25:25.99107831 +0800 CST StartTs:457118718606966785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00393eea0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118718620336129} {CheckpointTs:457118726340476936 MinTableBarrierTs:457118726340476936 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118726340476937, checkpointTs: 457118726340476936, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b {"id":"1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c3273441 1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b /tidb/cdc/default/default/changefeed/info/6fc6677d-b916-4c30-8c4b-61e64559c562 {"upstream-id":7489433030786483696,"namespace":"default","changefeed-id":"6fc6677d-b916-4c30-8c4b-61e64559c562","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:25:25.99107831+08:00","start-ts":457118718606966785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118718620336129} /tidb/cdc/default/default/changefeed/status/6fc6677d-b916-4c30-8c4b-61e64559c562 {"checkpoint-ts":457118726340476936,"min-table-barrier-ts":457118726340476936,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b/6fc6677d-b916-4c30-8c4b-61e64559c562 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433030786483696 {"id":7489433030786483696,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/6fc6677d-b916-4c30-8c4b-61e64559c562 {UpstreamID:7489433030786483696 Namespace:default ID:6fc6677d-b916-4c30-8c4b-61e64559c562 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:25:25.99107831 +0800 CST StartTs:457118718606966785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00393eea0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118718620336129} {CheckpointTs:457118726340476936 MinTableBarrierTs:457118726340476936 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118726340476937, checkpointTs: 457118726340476936, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b {"id":"1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c3273441 1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b /tidb/cdc/default/default/changefeed/info/6fc6677d-b916-4c30-8c4b-61e64559c562 {"upstream-id":7489433030786483696,"namespace":"default","changefeed-id":"6fc6677d-b916-4c30-8c4b-61e64559c562","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:25:25.99107831+08:00","start-ts":457118718606966785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118718620336129} /tidb/cdc/default/default/changefeed/status/6fc6677d-b916-4c30-8c4b-61e64559c562 {"checkpoint-ts":457118726340476936,"min-table-barrier-ts":457118726340476936,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b/6fc6677d-b916-4c30-8c4b-61e64559c562 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433030786483696 {"id":7489433030786483696,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/6fc6677d-b916-4c30-8c4b-61e64559c562 {UpstreamID:7489433030786483696 Namespace:default ID:6fc6677d-b916-4c30-8c4b-61e64559c562 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:25:25.99107831 +0800 CST StartTs:457118718606966785 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00393eea0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118718620336129} {CheckpointTs:457118726340476936 MinTableBarrierTs:457118726340476936 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118726340476937, checkpointTs: 457118726340476936, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118726340476936, checkpointTs: 457118726340476936, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b {"id":"1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c3273441 1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b /tidb/cdc/default/default/changefeed/info/6fc6677d-b916-4c30-8c4b-61e64559c562 {"upstream-id":7489433030786483696,"namespace":"default","changefeed-id":"6fc6677d-b916-4c30-8c4b-61e64559c562","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:25:25.99107831+08:00","start-ts":457118718606966785,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118718620336129} /tidb/cdc/default/default/changefeed/status/6fc6677d-b916-4c30-8c4b-61e64559c562 {"checkpoint-ts":457118726340476936,"min-table-barrier-ts":457118726340476936,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e8db9af-9e0e-41e0-a6c8-425feb0f5b4b/6fc6677d-b916-4c30-8c4b-61e64559c562 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433030786483696 {"id":7489433030786483696,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118727272136705, "checkpoint_time": "2025-04-04 20:25:58.991", "error": null }' + echo '{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118727272136705, "checkpoint_time": "2025-04-04 20:25:58.991", "error": null }' { "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118727272136705, "checkpoint_time": "2025-04-04 20:25:58.991", "error": null } ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118727272136705, '"checkpoint_time":' '"2025-04-04' '20:25:58.991",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118727272136705, '"checkpoint_time":' '"2025-04-04' '20:25:58.991",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40524.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x curl http://10.233.90.94:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13602; Connection refused run task failed 4-th time, retry later [2025/04/04 20:26:02.486 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m6.595648848s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark_5 not exists for 1-th check, retry later time="2025-04-04T20:26:04+08:00" level=info msg="Listening on 127.0.0.1:13602" time="2025-04-04T20:26:04+08:00" level=info msg="Not intercepting TLS connections" table mark.finish_mark_5 not exists for 2-th check, retry later + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457118727534018561, "checkpoint_time": "2025-04-04 20:25:59.990", "error": null }' + echo '{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457118727534018561, "checkpoint_time": "2025-04-04 20:25:59.990", "error": null }' { "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457118727534018561, "checkpoint_time": "2025-04-04 20:25:59.990", "error": null } ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457118727534018561, '"checkpoint_time":' '"2025-04-04' '20:25:59.990",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457118727534018561, '"checkpoint_time":' '"2025-04-04' '20:25:59.990",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40604.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489433147551337604,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:25:53.631986149+08:00","start_ts":457118723955752987,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457118727534018561,"checkpoint_time":"2025-04-04 20:25:59.990"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... table ci_collation_compatibility.t exists table ci_collation_compatibility.finish_mark_1 exists check diff successfully + set +x wait process cdc.test exit for 1-th time... table mark.finish_mark_5 exists table mark.finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40646.out cli changefeed --changefeed-id custom-changefeed-name resume wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:26:08 CST 2025] <<<<<< run test case ci_collation_compatibility success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + set +x curl http://10.233.90.94:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 15127 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 9225 started proxy pid: 9225 + echo started proxy at port: 13602 started proxy at port: 13602 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.90.94 2379 + pd_host=10.233.90.94 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://10.233.90.94:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12933.out cli tso query --pd=http://10.233.90.94:2379 + set +x + tso='457118730285219841 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118730285219841 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118730285219841 + echo 'query start ts: 457118730285219841' query start ts: 457118730285219841 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Fri Apr 4 20:26:11 CST 2025] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.1297412976.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 13602 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 13602 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Fri, 04 Apr 2025 12:26:11 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118730679746563, "checkpoint_time": "2025-04-04 20:26:11.990", "error": null }' + echo '{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118730679746563, "checkpoint_time": "2025-04-04 20:26:11.990", "error": null }' { "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118730679746563, "checkpoint_time": "2025-04-04 20:26:11.990", "error": null } ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118730679746563, '"checkpoint_time":' '"2025-04-04' '20:26:11.990",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118730679746563, '"checkpoint_time":' '"2025-04-04' '20:26:11.990",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40716.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457118730941890564 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 13602 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 13602 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 823 < Content-Type: text/plain; charset=utf-8 < Date: Fri, 04 Apr 2025 12:26:15 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f2c1783-6ee1-4fa8-9532-b55558df24b2 {"id":"3f2c1783-6ee1-4fa8-9532-b55558df24b2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769572} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/4a649600c37afb5f 3f2c1783-6ee1-4fa8-9532-b55558df24b2 /tidb/cdc/default/default/upstream/7489433126539382157 {"id":7489433126539382157,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.90.94:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f2c1783-6ee1-4fa8-9532-b55558df24b2 {"id":"3f2c1783-6ee1-4fa8-9532-b55558df24b2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769572} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/4a649600c37afb5f 3f2c1783-6ee1-4fa8-9532-b55558df24b2 /tidb/cdc/default/default/upstream/7489433126539382157 {"id":7489433126539382157,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.90.94:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f2c1783-6ee1-4fa8-9532-b55558df24b2 {"id":"3f2c1783-6ee1-4fa8-9532-b55558df24b2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769572} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/4a649600c37afb5f 3f2c1783-6ee1-4fa8-9532-b55558df24b2 /tidb/cdc/default/default/upstream/7489433126539382157 {"id":7489433126539382157,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.90.94:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=457118730285219841 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.13035.out cli changefeed create --start-ts=457118730285219841 --sink-uri=blackhole:/// Create changefeed successfully! ID: 3ea706aa-1e77-49d7-9607-184bc27d458b Info: {"upstream_id":7489433126539382157,"namespace":"default","id":"3ea706aa-1e77-49d7-9607-184bc27d458b","sink_uri":"blackhole:///","create_time":"2025-04-04T20:26:15.393776883+08:00","start_ts":457118730285219841,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118730285219841,"checkpoint_ts":457118730285219841,"checkpoint_time":"2025-04-04 20:26:10.485"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 5 The 1 times to try to start tidb cluster... changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40791.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name check_redo_resolved_ts 6fc6677d-b916-4c30-8c4b-61e64559c562 457118732002787335 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/6fc6677d-b916-4c30-8c4b-61e64559c562/meta global resolved ts 457118731596201987 not forward to 457118732002787335 run task failed 1-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489433147551337604,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:26:18.93040654+08:00","start_ts":457118732475695108,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118732475695108,"checkpoint_ts":457118732475695108,"checkpoint_time":"2025-04-04 20:26:18.841"} PASS [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] // container [Pipeline] } check_redo_resolved_ts 6fc6677d-b916-4c30-8c4b-61e64559c562 457118732002787335 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/6fc6677d-b916-4c30-8c4b-61e64559c562/meta run task successfully [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2025/04/04 20:26:20.915 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:26:20.916 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:26:20.916 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:26:20.916 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:26:20.917 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118726720585732] [resolvedTs=457118732120752131] [2025/04/04 20:26:20.917 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:26:20.918 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:26:20.918 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:26:20.918 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.059805ms] [2025/04/04 20:26:20.918 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.182402ms] [2025/04/04 20:26:20.923 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:26:20.926 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:26:20.928 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:26:20.928 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:26:20.928 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:26:20.928 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:26:20.933 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:26:20.935 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:26:20.935 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/04 20:26:20.935 +08:00] [ERROR] [redo.go:272] ["ignore unsupported DDL"] [ddl="{\"StartTs\":457118726707216417,\"CommitTs\":457118726720585732,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file_large_value`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file_large_value`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file_large_value\",\"Table\":\"t1\",\"TableID\":118,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:26:21.137 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118732120752131] [2025/04/04 20:26:21.137 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118732120752131] [2025/04/04 20:26:21.438 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118732120752131] [2025/04/04 20:26:21.438 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118732120752131] [2025/04/04 20:26:21.538 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=8] [appliedDDLCount=0] [currentCheckpoint=457118732120752131] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/04 20:26:21.538 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] Apply redo log successfully check diff successfully [Fri Apr 4 20:26:22 CST 2025] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq + service_type_count=31 + echo 'captured services: ' captured services: + echo . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager /tikvpb.Tikv '[2025/04' '[metadata=MD{:authority=[10.233.90.94:2379],' 'user-agent=[grpc-go/1.64.0],' 'grpc-accept-encoding=[gzip],' 'content-type=[application' '[metadata=MD{content-type=[application/grpc],' ':authority=[10.233.90.94:20160],' 'user-agent=[grpc-go' '[metadata=MD{content-type=[application/grpc],' ':authority=[10.233.90.94:2379],' 'user-agent=[grpc-go' '[metadata=MD{features=[stream-multiplexing],' 'user-agent=[grpc-go/1.64.0],' 'content-type=[application' '[metadata=MD{grpc-accept-encoding=[gzip],' 'content-type=[application/grpc],' ':authority=[10.233.90.94:2379],' 'user-agent=[grpc-go' '[metadata=MD{user-agent=[grpc-go/1.64.0],' 'grpc-accept-encoding=[gzip],' 'content-type=[application' '[method=/cdcpb.ChangeData' '[method=/pdpb.PD' '[method=/resource_manager.ResourceManager' '[stack=main.intercept\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go:56\ngithub.com/bradleyjkemp/grpc-tools/grpc-proxy.recoverWrapper.func1\n\t/go/pkg/mod/github.com/bradleyjkemp/grpc-tools@v0.2.5/grpc-proxy/config.go:52\ngoogle.golang.org/grpc.(*Server).processStreamingRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1680\ngoogle.golang.org/grpc.(*Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1800\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0' . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager /tikvpb.Tikv [2025/04 [metadata=MD{:authority=[10.233.90.94:2379], user-agent=[grpc-go/1.64.0], grpc-accept-encoding=[gzip], content-type=[application [metadata=MD{content-type=[application/grpc], :authority=[10.233.90.94:20160], user-agent=[grpc-go [metadata=MD{content-type=[application/grpc], :authority=[10.233.90.94:2379], user-agent=[grpc-go [metadata=MD{features=[stream-multiplexing], user-agent=[grpc-go/1.64.0], content-type=[application [metadata=MD{grpc-accept-encoding=[gzip], content-type=[application/grpc], :authority=[10.233.90.94:2379], user-agent=[grpc-go [metadata=MD{user-agent=[grpc-go/1.64.0], grpc-accept-encoding=[gzip], content-type=[application [method=/cdcpb.ChangeData [method=/pdpb.PD [method=/resource_manager.ResourceManager [stack=main.intercept\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go:56\ngithub.com/bradleyjkemp/grpc-tools/grpc-proxy.recoverWrapper.func1\n\t/go/pkg/mod/github.com/bradleyjkemp/grpc-tools@v0.2.5/grpc-proxy/config.go:52\ngoogle.golang.org/grpc.(*Server).processStreamingRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1680\ngoogle.golang.org/grpc.(*Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1800\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0 + '[' 31 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Fri Apr 4 20:26:21 CST 2025] <<<<<< run test case http_proxies success! >>>>>>' [Fri Apr 4 20:26:21 CST 2025] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118733563592709, "checkpoint_time": "2025-04-04 20:26:22.991", "error": null }' + echo '{ "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118733563592709, "checkpoint_time": "2025-04-04 20:26:22.991", "error": null }' { "upstream_id": 7489433147551337604, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457118733563592709, "checkpoint_time": "2025-04-04 20:26:22.991", "error": null } ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118733563592709, '"checkpoint_time":' '"2025-04-04' '20:26:22.991",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489433147551337604, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457118733563592709, '"checkpoint_time":' '"2025-04-04' '20:26:22.991",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40894.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.40923.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433332025153442 TiDB Cluster ID. tikv_gc_leader_uuid 65803114d6c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:27396, start at 2025-04-04 20:26:28.572397635 +0800 CST m=+2.403218376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:28.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:28.571 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:28.571 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433332025153442 TiDB Cluster ID. tikv_gc_leader_uuid 65803114d6c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:27396, start at 2025-04-04 20:26:28.572397635 +0800 CST m=+2.403218376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:28.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:28.571 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:28.571 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433323338673823 TiDB Cluster ID. tikv_gc_leader_uuid 65803114e380006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:27467, start at 2025-04-04 20:26:28.666817362 +0800 CST m=+2.445840019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:28.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:28.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:28.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41018.out cli unsafe resolve-lock --region=4 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.28786.out cli tso query --pd=http://127.0.0.1:2379 Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.41053.out cli unsafe resolve-lock --region=4 --ts=457118734979170306 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18331D34E3033492 < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 12:26:33 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... + stop_proxy + kill 9225 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + tso='457118736153837570 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118736153837570 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:26:34 CST 2025] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.2882328825.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1774 50 --:--:-- --:--:-- --:--:-- 1781 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:26:37 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57caad5d-dc22-4a01-822a-1ba100bff7e7 {"id":"57caad5d-dc22-4a01-822a-1ba100bff7e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c43220e5 57caad5d-dc22-4a01-822a-1ba100bff7e7 /tidb/cdc/default/default/upstream/7489433332025153442 {"id":7489433332025153442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57caad5d-dc22-4a01-822a-1ba100bff7e7 {"id":"57caad5d-dc22-4a01-822a-1ba100bff7e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c43220e5 57caad5d-dc22-4a01-822a-1ba100bff7e7 /tidb/cdc/default/default/upstream/7489433332025153442 {"id":7489433332025153442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57caad5d-dc22-4a01-822a-1ba100bff7e7 {"id":"57caad5d-dc22-4a01-822a-1ba100bff7e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c43220e5 57caad5d-dc22-4a01-822a-1ba100bff7e7 /tidb/cdc/default/default/upstream/7489433332025153442 {"id":7489433332025153442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.28882.out cli changefeed create --start-ts=457118736153837570 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 6b5c0d16-3cd2-49d1-abd5-d3d31ff93e84 Info: {"upstream_id":7489433332025153442,"namespace":"default","id":"6b5c0d16-3cd2-49d1-abd5-d3d31ff93e84","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:26:37.781820118+08:00","start_ts":457118736153837570,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118736153837570,"checkpoint_ts":457118736153837570,"checkpoint_time":"2025-04-04 20:26:32.872"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 1551 0 --:--:-- --:--:-- --:--:-- 1555 { "version": "v9.0.0-beta.1-9-g4c631d595", "git_hash": "4c631d5951854ac91765b729fe3849d8e8d5ed8a", "id": "88420182-910a-4d0f-b495-e6a216067592", "pid": 40284, "is_owner": true }wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:26:39 CST 2025] <<<<<< run test case cli_tls_with_auth success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:26:45.041 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2025/04/04 20:26:45.275 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:26:45.280 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:26:45.480 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:26:45.481 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:26:45.483 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:26:45.484 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433401468329032 TiDB Cluster ID. tikv_gc_leader_uuid 658031242a00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:14077, start at 2025-04-04 20:26:44.265377405 +0800 CST m=+2.151262815 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:44.273 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:44.264 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:44.264 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433401468329032 TiDB Cluster ID. tikv_gc_leader_uuid 658031242a00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:14077, start at 2025-04-04 20:26:44.265377405 +0800 CST m=+2.151262815 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:44.273 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:44.264 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:44.264 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433397216475628 TiDB Cluster ID. tikv_gc_leader_uuid 65803124414000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-bzrcn, pid:14157, start at 2025-04-04 20:26:44.393652177 +0800 CST m=+2.223427274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:44.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:44.428 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:44.428 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sequence/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sequence/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sequence/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:26:46.711 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:26:46.718 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:26:46.924 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:26:46.926 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:26:46.931 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:26:46.932 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:26:41.267 +08:00] [WARN] [diff.go:766] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m31.344801666s] [2025/04/04 20:26:41.270 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2025/04/04 20:26:46.833 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] [2025/04/04 20:26:47.133 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:26:47.137 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433390891671916 TiDB Cluster ID. tikv_gc_leader_uuid 658031251e80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:16009, start at 2025-04-04 20:26:45.24385759 +0800 CST m=+2.120682904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:45.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:45.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:45.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433390891671916 TiDB Cluster ID. tikv_gc_leader_uuid 658031251e80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:16009, start at 2025-04-04 20:26:45.24385759 +0800 CST m=+2.120682904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:45.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:45.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:45.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433406480923177 TiDB Cluster ID. tikv_gc_leader_uuid 6580312542c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:16073, start at 2025-04-04 20:26:45.416574412 +0800 CST m=+2.238052674 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:28:45.441 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:26:45.438 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:16:45.438 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2025/04/04 20:26:48.358 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:26:48.359 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:26:48.367 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:26:48.368 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:26:48 CST 2025] <<<<<< run test case cdc success! >>>>>> [2025/04/04 20:26:48.697 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:26:48.709 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:26:48.904 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:26:48.905 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [Fri Apr 4 20:26:48 CST 2025] <<<<<< START cdc server in sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.1559515597.out server --log-file /tmp/tidb_cdc_test/sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/debezium_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:26:49 CST 2025] <<<<<< run test case debezium_basic success! >>>>>> [Fri Apr 4 20:26:49 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1734917351.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/04 20:26:49.937 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:26:49.940 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:26:51 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11d58933-61a6-460a-9f44-6417d4122abd {"id":"11d58933-61a6-460a-9f44-6417d4122abd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769608} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c47643d8 11d58933-61a6-460a-9f44-6417d4122abd /tidb/cdc/default/default/upstream/7489433401468329032 {"id":7489433401468329032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11d58933-61a6-460a-9f44-6417d4122abd {"id":"11d58933-61a6-460a-9f44-6417d4122abd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769608} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c47643d8 11d58933-61a6-460a-9f44-6417d4122abd /tidb/cdc/default/default/upstream/7489433401468329032 {"id":7489433401468329032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11d58933-61a6-460a-9f44-6417d4122abd {"id":"11d58933-61a6-460a-9f44-6417d4122abd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769608} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c47643d8 11d58933-61a6-460a-9f44-6417d4122abd /tidb/cdc/default/default/upstream/7489433401468329032 {"id":7489433401468329032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.cli.15653.out cli changefeed create --start-ts=457118740254294018 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/conf/force_replicate.toml Create changefeed successfully! ID: 80e8519a-0a96-4408-8394-b88e166cd92d Info: {"upstream_id":7489433401468329032,"namespace":"default","id":"80e8519a-0a96-4408-8394-b88e166cd92d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:26:52.083438142+08:00","start_ts":457118740254294018,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118740254294018,"checkpoint_ts":457118740254294018,"checkpoint_time":"2025-04-04 20:26:48.514"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:26:52 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20f41bdc-7589-46b0-bebd-af68771a48fb {"id":"20f41bdc-7589-46b0-bebd-af68771a48fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4762dda 20f41bdc-7589-46b0-bebd-af68771a48fb /tidb/cdc/default/default/upstream/7489433390891671916 {"id":7489433390891671916,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20f41bdc-7589-46b0-bebd-af68771a48fb {"id":"20f41bdc-7589-46b0-bebd-af68771a48fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4762dda 20f41bdc-7589-46b0-bebd-af68771a48fb /tidb/cdc/default/default/upstream/7489433390891671916 {"id":7489433390891671916,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20f41bdc-7589-46b0-bebd-af68771a48fb {"id":"20f41bdc-7589-46b0-bebd-af68771a48fb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4762dda 20f41bdc-7589-46b0-bebd-af68771a48fb /tidb/cdc/default/default/upstream/7489433390891671916 {"id":7489433390891671916,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "mysql.host"="127.0.0.1" "threadcount"="10" "fieldcount"="100" "scanproportion"="0" "mysql.port"="4000" "readallfields"="true" "insertproportion"="0" "dotransactions"="false" "operationcount"="0" "updateproportion"="0" "mysql.db"="consistent_replicate_storage_s3" "readproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "recordcount"="5000" "workload"="core" ********************************************** + set +x table sequence_test.t1 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:26:53 CST 2025] <<<<<< run test case kafka_simple_basic success! >>>>>> Run finished, takes 1.904715694s INSERT - Takes(s): 1.9, Count: 5000, OPS: 2658.1, Avg(us): 3453, Min(us): 1233, Max(us): 57861, 95th(us): 4000, 99th(us): 48000 table consistent_replicate_storage_s3.usertable not exists for 1-th check, retry later [2025/04/04 20:26:55.551 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2025/04/04 20:26:55.785 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:26:55.794 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:26:55.987 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:26:55.989 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:26:56.010 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:26:56.012 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] table sequence_test.t1 exists check table exists success check diff failed 1-th time, retry later [2025/04/04 20:26:56.185 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:26:56.228 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:26:56.387 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:26:56.393 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:26:56.438 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:26:56.444 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:26:56.616 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:26:56.691 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:26:56.829 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later [2025/04/04 20:26:56.836 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:26:56.914 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:26:56.920 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:26:57.056 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:26:56 CST 2025] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> [2025/04/04 20:26:57.120 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:26:57.270 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:26:57.277 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:26:57.326 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:26:57.336 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:26:57.487 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:26:57.542 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:26:57.702 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:26:57.713 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:26:57.749 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:26:57.761 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] check diff successfully ERROR 1146 (42S02) at line 1: Table 'sequence_test.seq0' doesn't exist table sequence_test.mark_table not exists for 1-th check, retry later [2025/04/04 20:26:57.911 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:26:57.962 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/04 20:26:58.120 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:26:58.131 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:26:58.166 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:26:58.176 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:26:58.328 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:26:58.362 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:26:58.552 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:26:58.563 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:26:58.563 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:26:58.574 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:26:58.767 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:26:58.768 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:26:58.974 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:26:58.979 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:26:58.984 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:26:58.989 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:26:59.171 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:26:59.187 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:26:59.371 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:26:59.386 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:26:59.391 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later [2025/04/04 20:26:59.402 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:26:59.720 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/04 20:26:59.758 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:26:59.929 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:26:59.947 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/04 20:26:59.957 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:26:59.967 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/04 20:27:00.126 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] [2025/04/04 20:27:00.155 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:00 CST 2025] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> [2025/04/04 20:27:00.307 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] [2025/04/04 20:27:00.323 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] [2025/04/04 20:27:00.344 +08:00] [INFO] [main.go:221] ["0 insert success: 2200"] [2025/04/04 20:27:00.353 +08:00] [INFO] [main.go:235] ["0 delete success: 1100"] table sequence_test.mark_table exists wait process cdc.test exit for 1-th time... [2025/04/04 20:27:00.490 +08:00] [INFO] [main.go:221] ["1 insert success: 2300"] [2025/04/04 20:27:00.546 +08:00] [INFO] [main.go:221] ["0 insert success: 2300"] wait process cdc.test exit for 2-th time... table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:27:01 CST 2025] <<<<<< run test case sequence success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:03 CST 2025] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> Verifying downstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7152 0 --:--:-- --:--:-- --:--:-- 7366 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 20:26:52.626","puller_resolved_ts":"2025-04-04 20:26:44.577","last_synced_ts":"2025-04-04 20:24:36.127","now_ts":"2025-04-04 20:26:53.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:26:52.626","puller_resolved_ts":"2025-04-04' '20:26:44.577","last_synced_ts":"2025-04-04' '20:24:36.127","now_ts":"2025-04-04' '20:26:53.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 14792 8.4 0.0 13657696 148696 ? Sl 20:24 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 14855 6.3 0.0 13648220 142680 ? Sl 20:24 0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1743769619.9696226,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc005425340/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743769619.9696908,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769619.969796,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0046cddc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769619.9698536,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769620.1146274,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0030d6c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769620.1146927,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:27:06.480 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2025/04/04 20:27:06.695 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:27:06.696 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:27:06.883 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:27:06.885 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:27:06.887 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:27:06.889 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:27:07.073 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:27:07.076 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:27:07.254 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:27:07.256 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:27:07.268 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:27:07.270 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:27:07.446 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later [2025/04/04 20:27:07.470 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:07 CST 2025] <<<<<< run test case kafka_simple_claim_check success! >>>>>> [2025/04/04 20:27:07.789 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:27:07.791 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:27:07.815 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:27:07.817 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:27:07.972 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] {"level":"warn","ts":"2025-04-04T20:27:07.458933+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:07.459379+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:07.567307+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:27:08.001 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:27:08.163 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:27:08.165 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:27:08.190 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:27:08.192 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:27:08.361 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:27:08.387 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:27:08.543 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:27:08.545 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:27:08.574 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:27:08.576 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:27:08.732 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:27:08.777 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:27:08.916 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:27:08.918 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:27:08.974 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:27:08.976 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:27:09.097 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:27:09.157 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:27:09.273 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:27:09.274 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/04 20:27:09.344 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:27:09.346 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:27:09.448 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later [2025/04/04 20:27:09.679 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:27:09.830 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:27:09.832 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:27:09.953 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:27:09.956 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:27:10.047 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:27:10.179 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:27:10.257 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:27:10.258 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:27:10.393 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:27:10.396 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:27:10.458 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/04 20:27:10.606 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:27:10.664 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:27:10.666 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/04 20:27:10.825 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:27:10.827 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/04 20:27:10.877 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] [2025/04/04 20:27:11.038 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:10 CST 2025] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> [2025/04/04 20:27:11.082 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] [2025/04/04 20:27:11.084 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] [2025/04/04 20:27:11.255 +08:00] [INFO] [main.go:221] ["0 insert success: 2200"] [2025/04/04 20:27:11.257 +08:00] [INFO] [main.go:235] ["0 delete success: 1100"] [2025/04/04 20:27:11.295 +08:00] [INFO] [main.go:221] ["1 insert success: 2300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433500950479714 TiDB Cluster ID. tikv_gc_leader_uuid 6580313c21c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:24600, start at 2025-04-04 20:27:08.841982112 +0800 CST m=+2.051297942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:08.855 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:08.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:08.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433500950479714 TiDB Cluster ID. tikv_gc_leader_uuid 6580313c21c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:24600, start at 2025-04-04 20:27:08.841982112 +0800 CST m=+2.051297942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:08.855 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:08.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:08.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433505602571596 TiDB Cluster ID. tikv_gc_leader_uuid 6580313c390000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-3-f6tqn, pid:24670, start at 2025-04-04 20:27:08.933589747 +0800 CST m=+2.080862195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:08.941 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:08.900 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:08.900 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.25939.out cli tso query --pd=http://127.0.0.1:2379 {"level":"warn","ts":1743769633.0547996,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0046cddc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":1743769633.058695,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc005425340/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":"2025-04-04T20:27:13.460302+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:13.460824+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:13.568997+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:13 CST 2025] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> + set +x + tso='457118746806059009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118746806059009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:27:14 CST 2025] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2599025992.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 table consistent_replicate_storage_s3.check1 not exists for 10-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:17 CST 2025] <<<<<< run test case canal_json_basic success! >>>>>> table consistent_replicate_storage_s3.check1 not exists for 11-th check, retry later [2025/04/04 20:27:17.429 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2025/04/04 20:27:17.636 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:27:17.650 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:27:17.835 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:27:17.837 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:27:17.846 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:27:17.848 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:27:18.040 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:27:18.051 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:27:18.248 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:27:18.250 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:27:18.257 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:27:18.259 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:27:18 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62 {"id":"5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769635} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4d492fc 5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62 /tidb/cdc/default/default/upstream/7489433500950479714 {"id":7489433500950479714,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62 {"id":"5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769635} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4d492fc 5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62 /tidb/cdc/default/default/upstream/7489433500950479714 {"id":7489433500950479714,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62 {"id":"5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769635} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4d492fc 5fb7babd-0aaf-49f2-a5e6-26b53f7dbf62 /tidb/cdc/default/default/upstream/7489433500950479714 {"id":7489433500950479714,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26044.out cli changefeed create --start-ts=457118746806059009 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 [2025/04/04 20:27:18.447 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:27:18.447 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] Create changefeed successfully! ID: 57d1e1f1-7019-468e-b31b-057c295e52b8 Info: {"upstream_id":7489433500950479714,"namespace":"default","id":"57d1e1f1-7019-468e-b31b-057c295e52b8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:27:18.436079287+08:00","start_ts":457118746806059009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118746806059009,"checkpoint_ts":457118746806059009,"checkpoint_time":"2025-04-04 20:27:13.507"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/04 20:27:18.643 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:27:18.645 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:27:18.646 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:27:18.647 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:27:18.835 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:27:18.841 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/04 20:27:19.021 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:27:19.023 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:27:19.035 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:27:19.037 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 [2025/04/04 20:27:19.218 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:27:19.234 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] table consistent_replicate_storage_s3.check1 not exists for 12-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:27:19.462256+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:19.46237+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:19.570566+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:27:19.416 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:27:19.418 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:27:19.433 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:27:19.435 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:27:19.784 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:27:19.796 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] + set +x table split_region.test1 not exists for 1-th check, retry later [2025/04/04 20:27:20.000 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:27:20.002 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:27:20.007 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:27:20.009 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:27:20.268 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:27:20.283 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:27:20.578 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:27:20.580 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:27:20.587 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:27:20.589 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:20 CST 2025] <<<<<< run test case canal_json_content_compatible success! >>>>>> [2025/04/04 20:27:20.771 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:27:20.859 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:27:21.131 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:27:21.133 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:27:21.165 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:27:21.167 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:27:21.321 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:27:21.362 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later [2025/04/04 20:27:21.702 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:27:21.704 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:27:21.745 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:27:21.747 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:27:21.897 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:27:21.945 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] table consistent_replicate_storage_s3.check1 not exists for 13-th check, retry later [2025/04/04 20:27:22.088 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:27:22.091 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 [2025/04/04 20:27:22.284 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/04 20:27:22.287 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 table consistent_replicate_storage_s3.check1 not exists for 14-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:24 CST 2025] <<<<<< run test case multi_topics success! >>>>>> table split_region.test2 exists check diff successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:27:25.462764+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:25.463828+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} table consistent_replicate_storage_s3.check1 not exists for 15-th check, retry later {"level":"warn","ts":"2025-04-04T20:27:25.572578+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:27 CST 2025] <<<<<< run test case avro_basic success! >>>>>> table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:27:27.447775+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T20:27:27.447823+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T20:27:27.44861+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T20:27:27.44866+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T20:27:27.557565+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-04T20:27:27.557617+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} [2025/04/04 20:27:28.302 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2025/04/04 20:27:28.592 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:27:28.594 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:27:28.908 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:27:28.910 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:27:28.921 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:27:28.923 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/04 20:27:29.175 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:27:29.193 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:27:29.432 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:27:29.434 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:27:29.447 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:27:29.449 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 [2025/04/04 20:27:29.631 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/04 20:27:29.649 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:27:30.009 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:27:30.011 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:27:30.037 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:27:30.039 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:27:30.209 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:27:30.249 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 [2025/04/04 20:27:30.466 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:27:30.468 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:27:30.518 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:27:30.520 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:27:30.686 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:27:30.751 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:27:30.883 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:27:30.885 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:27:31.013 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:27:31.015 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:27:31.182 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/04 20:27:31.251 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 [2025/04/04 20:27:31.380 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:27:31.382 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:27:31.450 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:27:31.452 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:27:31.463883+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:31.465105+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:31.626333+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:30 CST 2025] <<<<<< run test case canal_json_handle_key_only success! >>>>>> [2025/04/04 20:27:31.674 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:27:31.749 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later [2025/04/04 20:27:31.874 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:27:31.876 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:27:31.949 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:27:31.951 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/04 20:27:32.069 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/04 20:27:32.200 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:27:32.352 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/04 20:27:32.354 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:27:32.446 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:27:32.449 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:27:32.577 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/04 20:27:32.685 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/04 20:27:32.898 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/04 20:27:32.900 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/04 20:27:33.009 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/04 20:27:33.011 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/04 20:27:33.091 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/04 20:27:33.207 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/04 20:27:33.275 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/04 20:27:33.276 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:34 CST 2025] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1743769654.9708517,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0046cddc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769654.9708955,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769654.9710813,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc005425340/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743769654.9711123,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743769655.115654,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0030d6c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743769655.1157017,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 check diff failed 1-th time, retry later ____________________________________ *************************** 1. row *************************** primary_ts: 457118688215040000 secondary_ts: 457118689840594990 *************************** 2. row *************************** primary_ts: 457118696079360000 secondary_ts: 457118696407302145 *************************** 3. row *************************** primary_ts: 457118703943680000 secondary_ts: 457118704061644801 *************************** 4. row *************************** primary_ts: 457118711808000000 secondary_ts: 457118711886643202 *************************** 5. row *************************** primary_ts: 457118719672320000 secondary_ts: 457118719816761345 *************************** 6. row *************************** primary_ts: 457118727536640000 secondary_ts: 457118727641759747 *************************** 7. row *************************** primary_ts: 457118735400960000 secondary_ts: 457118735505817603 *************************** 8. row *************************** primary_ts: 457118743265280000 secondary_ts: 457118743383506946 skip invalid syncpoint primary_ts: 457118688215040000, first_ddl_ts: 457118724680056835 skip invalid syncpoint primary_ts: 457118696079360000, first_ddl_ts: 457118724680056835 skip invalid syncpoint primary_ts: 457118703943680000, first_ddl_ts: 457118724680056835 skip invalid syncpoint primary_ts: 457118711808000000, first_ddl_ts: 457118724680056835 skip invalid syncpoint primary_ts: 457118719672320000, first_ddl_ts: 457118724680056835 check diff successfully check diff successfully check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:27:29 CST 2025] <<<<<< run test case syncpoint success! >>>>>> table consistent_replicate_storage_s3.t2 exists 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0{"level":"warn","ts":"2025-04-04T20:27:37.464869+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:37.466032+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:37.627392+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:37 CST 2025] <<<<<< run test case canal_json_claim_check success! >>>>>> check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint_check_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 wait process cdc.test exit for 3-th time... [2025/04/04 20:27:39.211 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2025/04/04 20:27:39.410 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/04 20:27:39.424 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/04 20:27:39.682 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/04 20:27:39.684 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/04 20:27:39.764 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/04 20:27:39.766 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:27:39 CST 2025] <<<<<< run test case split_region success! >>>>>> [2025/04/04 20:27:39.925 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/04 20:27:39.952 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/04 20:27:40.166 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/04 20:27:40.167 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/04 20:27:40.262 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/04 20:27:40.264 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/04 20:27:40.411 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 20:27:40 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1770117703.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 [2025/04/04 20:27:40.476 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/04 20:27:40.792 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/04 20:27:40.794 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/04 20:27:40.872 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/04 20:27:40.874 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/04 20:27:41.048 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/04 20:27:41.160 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:40 CST 2025] <<<<<< run test case open_protocol_claim_check success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 [2025/04/04 20:27:41.316 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/04 20:27:41.318 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/04 20:27:41.441 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/04 20:27:41.452 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/04 20:27:41.663 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/04 20:27:41.794 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/04 20:27:41.936 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/04 20:27:41.937 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/04 20:27:42.036 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/04 20:27:42.057 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/04 20:27:42.187 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] start tidb cluster in /tmp/tidb_cdc_test/syncpoint_check_ts Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 [2025/04/04 20:27:42.307 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/04 20:27:42.425 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/04 20:27:42.426 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/04 20:27:42.608 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/04 20:27:42.612 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/04 20:27:42.740 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/04 20:27:42.901 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/04 20:27:43.014 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/04 20:27:43.015 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/04 20:27:43.191 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/04 20:27:43.195 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/04 20:27:43.333 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:27:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {UpstreamID:7489433390891671916 Namespace:default ID:e90858a3-4b58-425f-82fe-bfaaed0e9d5a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:26:52.714140912 +0800 CST StartTs:457118741336424451 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019d6990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118741349531651} {CheckpointTs:457118754364981254 MinTableBarrierTs:457118754364981254 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47fa0748-463c-4a04-8701-9effe17ff287 {"id":"47fa0748-463c-4a04-8701-9effe17ff287","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769660} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4762fd1 47fa0748-463c-4a04-8701-9effe17ff287 /tidb/cdc/default/default/changefeed/info/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"upstream-id":7489433390891671916,"namespace":"default","changefeed-id":"e90858a3-4b58-425f-82fe-bfaaed0e9d5a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:26:52.714140912+08:00","start-ts":457118741336424451,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118741349531651} /tidb/cdc/default/default/changefeed/status/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"checkpoint-ts":457118754364981254,"min-table-barrier-ts":457118754364981254,"admin-job-type":0} /tidb/cdc/default/default/task/position/47fa0748-463c-4a04-8701-9effe17ff287/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433390891671916 {"id":7489433390891671916,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {UpstreamID:7489433390891671916 Namespace:default ID:e90858a3-4b58-425f-82fe-bfaaed0e9d5a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:26:52.714140912 +0800 CST StartTs:457118741336424451 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019d6990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118741349531651} {CheckpointTs:457118754364981254 MinTableBarrierTs:457118754364981254 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47fa0748-463c-4a04-8701-9effe17ff287 {"id":"47fa0748-463c-4a04-8701-9effe17ff287","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769660} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4762fd1 47fa0748-463c-4a04-8701-9effe17ff287 /tidb/cdc/default/default/changefeed/info/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"upstream-id":7489433390891671916,"namespace":"default","changefeed-id":"e90858a3-4b58-425f-82fe-bfaaed0e9d5a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:26:52.714140912+08:00","start-ts":457118741336424451,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118741349531651} /tidb/cdc/default/default/changefeed/status/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"checkpoint-ts":457118754364981254,"min-table-barrier-ts":457118754364981254,"admin-job-type":0} /tidb/cdc/default/default/task/position/47fa0748-463c-4a04-8701-9effe17ff287/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433390891671916 {"id":7489433390891671916,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {UpstreamID:7489433390891671916 Namespace:default ID:e90858a3-4b58-425f-82fe-bfaaed0e9d5a SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:26:52.714140912 +0800 CST StartTs:457118741336424451 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019d6990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118741349531651} {CheckpointTs:457118754364981254 MinTableBarrierTs:457118754364981254 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118754364981255, checkpointTs: 457118754364981254, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47fa0748-463c-4a04-8701-9effe17ff287 {"id":"47fa0748-463c-4a04-8701-9effe17ff287","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769660} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c4762fd1 47fa0748-463c-4a04-8701-9effe17ff287 /tidb/cdc/default/default/changefeed/info/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"upstream-id":7489433390891671916,"namespace":"default","changefeed-id":"e90858a3-4b58-425f-82fe-bfaaed0e9d5a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:26:52.714140912+08:00","start-ts":457118741336424451,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118741349531651} /tidb/cdc/default/default/changefeed/status/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"checkpoint-ts":457118754364981254,"min-table-barrier-ts":457118754364981254,"admin-job-type":0} /tidb/cdc/default/default/task/position/47fa0748-463c-4a04-8701-9effe17ff287/e90858a3-4b58-425f-82fe-bfaaed0e9d5a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433390891671916 {"id":7489433390891671916,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test {"level":"warn","ts":"2025-04-04T20:27:43.465177+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:43.466168+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f04c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T20:27:43.628617+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020048c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [2025/04/04 20:27:43.531 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/04 20:27:43.672 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/04 20:27:43.673 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 24548 0 --:--:-- --:--:-- --:--:-- 24931 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:25:21.422","puller_resolved_ts":"2025-04-04 20:25:21.422","last_synced_ts":"2025-04-04 20:25:14.571","now_ts":"2025-04-04 20:27:31.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:25:21.422","puller_resolved_ts":"2025-04-04' '20:25:21.422","last_synced_ts":"2025-04-04' '20:25:14.571","now_ts":"2025-04-04' '20:27:31.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:25:21.422","puller_resolved_ts":"2025-04-04' '20:25:21.422","last_synced_ts":"2025-04-04' '20:25:14.571","now_ts":"2025-04-04' '20:27:31.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory wait process cdc.test exit for 1-th time... [2025/04/04 20:27:43.939 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/04 20:27:43.943 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/04 20:27:44.041 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:44 CST 2025] <<<<<< run test case canal_json_storage_basic success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:27:47 CST 2025] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433682315605127 TiDB Cluster ID. tikv_gc_leader_uuid 658031643040007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:12638, start at 2025-04-04 20:27:49.836112608 +0800 CST m=+2.269861878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:49.844 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:49.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:49.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433682315605127 TiDB Cluster ID. tikv_gc_leader_uuid 658031643040007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:12638, start at 2025-04-04 20:27:49.836112608 +0800 CST m=+2.269861878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:49.844 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:49.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:49.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433673195450154 TiDB Cluster ID. tikv_gc_leader_uuid 658031641c4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:12715, start at 2025-04-04 20:27:49.778378856 +0800 CST m=+2.139099434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:49.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:49.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:49.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Logging trace to /tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/syncpoint_check_ts/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/04 20:27:51.695 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m6.653609372s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark_5 exists table mark.finish_mark not exists for 1-th check, retry later You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18331D4768A14FE3 < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 12:27:53 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint_check_ts.cli.13969.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433693542605945 TiDB Cluster ID. tikv_gc_leader_uuid 65803166dd40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:15383, start at 2025-04-04 20:27:52.61315409 +0800 CST m=+2.403494923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:52.659 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:52.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:52.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433693542605945 TiDB Cluster ID. tikv_gc_leader_uuid 65803166dd40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:15383, start at 2025-04-04 20:27:52.61315409 +0800 CST m=+2.403494923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:52.659 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:52.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:52.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433689709075363 TiDB Cluster ID. tikv_gc_leader_uuid 65803166f50000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:15458, start at 2025-04-04 20:27:52.683199952 +0800 CST m=+2.392541444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:29:52.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:27:52.660 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:17:52.660 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... wait process cdc.test exit for 2-th time... + set +x + tso='457118757487378433 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118757487378433 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:27:55 CST 2025] <<<<<< START cdc server in syncpoint_check_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint_check_ts.1400514007.out server --log-file /tmp/tidb_cdc_test/syncpoint_check_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint_check_ts/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:27:56 CST 2025] <<<<<< run test case multi_source success! >>>>>> + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16797.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457118758185467905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118758185467905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118758185467905 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 20:27:58 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1682716829.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:27:58 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44b2694f-f166-48b2-bb62-9aa5a8a11017 {"id":"44b2694f-f166-48b2-bb62-9aa5a8a11017","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769676} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c571f9e6 44b2694f-f166-48b2-bb62-9aa5a8a11017 /tidb/cdc/default/default/upstream/7489433682315605127 {"id":7489433682315605127,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44b2694f-f166-48b2-bb62-9aa5a8a11017 {"id":"44b2694f-f166-48b2-bb62-9aa5a8a11017","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769676} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c571f9e6 44b2694f-f166-48b2-bb62-9aa5a8a11017 /tidb/cdc/default/default/upstream/7489433682315605127 {"id":7489433682315605127,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44b2694f-f166-48b2-bb62-9aa5a8a11017 {"id":"44b2694f-f166-48b2-bb62-9aa5a8a11017","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769676} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c571f9e6 44b2694f-f166-48b2-bb62-9aa5a8a11017 /tidb/cdc/default/default/upstream/7489433682315605127 {"id":7489433682315605127,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint_check_ts.cli.14060.out cli changefeed create --start-ts=457118757487378433 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint_check_ts/conf/changefeed.toml Create changefeed successfully! ID: e85ec803-0601-4a02-92d3-b9f1f3709e2f Info: {"upstream_id":7489433682315605127,"namespace":"default","id":"e85ec803-0601-4a02-92d3-b9f1f3709e2f","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:27:59.13880142+08:00","start_ts":457118757487378433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118757487378433,"checkpoint_ts":457118757487378433,"checkpoint_time":"2025-04-04 20:27:54.253"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:28:01 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba02ff6c-6743-440b-a41f-4bf2a57d50f0 {"id":"ba02ff6c-6743-440b-a41f-4bf2a57d50f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769678} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c57e55e5 ba02ff6c-6743-440b-a41f-4bf2a57d50f0 /tidb/cdc/default/default/upstream/7489433693542605945 {"id":7489433693542605945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba02ff6c-6743-440b-a41f-4bf2a57d50f0 {"id":"ba02ff6c-6743-440b-a41f-4bf2a57d50f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769678} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c57e55e5 ba02ff6c-6743-440b-a41f-4bf2a57d50f0 /tidb/cdc/default/default/upstream/7489433693542605945 {"id":7489433693542605945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba02ff6c-6743-440b-a41f-4bf2a57d50f0 {"id":"ba02ff6c-6743-440b-a41f-4bf2a57d50f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769678} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c57e55e5 ba02ff6c-6743-440b-a41f-4bf2a57d50f0 /tidb/cdc/default/default/upstream/7489433693542605945 {"id":7489433693542605945,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118758185467905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16886.out cli changefeed create --start-ts=457118758185467905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489433693542605945,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:28:01.933527476+08:00","start_ts":457118758185467905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118758185467905,"checkpoint_ts":457118758185467905,"checkpoint_time":"2025-04-04 20:27:56.916"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433733051773213 TiDB Cluster ID. tikv_gc_leader_uuid 65803170378000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:18266, start at 2025-04-04 20:28:02.177333811 +0800 CST m=+2.328559543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:02.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:02.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:02.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433733051773213 TiDB Cluster ID. tikv_gc_leader_uuid 65803170378000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:18266, start at 2025-04-04 20:28:02.177333811 +0800 CST m=+2.328559543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:02.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:02.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:02.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433736747764594 TiDB Cluster ID. tikv_gc_leader_uuid 6580317036c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:18341, start at 2025-04-04 20:28:02.167507012 +0800 CST m=+2.228993351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:02.175 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:02.139 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:02.139 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_redo_resolved_ts e90858a3-4b58-425f-82fe-bfaaed0e9d5a 457118760158363654 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/e90858a3-4b58-425f-82fe-bfaaed0e9d5a/meta global resolved ts 457118759634337795 not forward to 457118760158363654 run task failed 1-th time, retry later table test.t1 exists + sleep 5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433739627791549 TiDB Cluster ID. tikv_gc_leader_uuid 6580317161c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:43564, start at 2025-04-04 20:28:03.377751923 +0800 CST m=+2.136330733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:03.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:03.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:03.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433739627791549 TiDB Cluster ID. tikv_gc_leader_uuid 6580317161c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:43564, start at 2025-04-04 20:28:03.377751923 +0800 CST m=+2.136330733 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:03.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:03.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:03.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433741280804785 TiDB Cluster ID. tikv_gc_leader_uuid 658031717940006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-8-9w94l, pid:43626, start at 2025-04-04 20:28:03.436497161 +0800 CST m=+2.137474865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:03.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:03.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:03.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/vector/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19592.out cli tso query --pd=http://127.0.0.1:2379 check_redo_resolved_ts e90858a3-4b58-425f-82fe-bfaaed0e9d5a 457118760158363654 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/e90858a3-4b58-425f-82fe-bfaaed0e9d5a/meta global resolved ts 457118760158363651 not forward to 457118760158363654 run task failed 2-th time, retry later [Fri Apr 4 20:28:07 CST 2025] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.4506145063.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457118760708866049 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118760708866049 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118760708866049 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 20:28:07 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1962719629.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/vector Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15383 10.1 0.0 2799232 213112 ? Sl 20:27 0:02 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743769670203267131.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15387 4.0 0.0 2815816 193044 ? Sl 20:27 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743769670206262226.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 15458 10.8 0.0 2743396 224420 ? Sl 20:27 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743769670284093225.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 14452 0 --:--:-- --:--:-- --:--:-- 15187 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:28:07.265","puller_resolved_ts":"2025-04-04 20:28:03.365","last_synced_ts":"2025-04-04 20:28:03.365","now_ts":"2025-04-04 20:28:10.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:28:07.265","puller_resolved_ts":"2025-04-04' '20:28:03.365","last_synced_ts":"2025-04-04' '20:28:03.365","now_ts":"2025-04-04' '20:28:10.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:28:07.265","puller_resolved_ts":"2025-04-04' '20:28:03.365","last_synced_ts":"2025-04-04' '20:28:03.365","now_ts":"2025-04-04' '20:28:10.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 check_redo_resolved_ts e90858a3-4b58-425f-82fe-bfaaed0e9d5a 457118760158363654 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/e90858a3-4b58-425f-82fe-bfaaed0e9d5a/meta run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:28:10 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50a74319-de1c-471c-9419-8f42dcacf3ae {"id":"50a74319-de1c-471c-9419-8f42dcacf3ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769688} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5ad48d8 50a74319-de1c-471c-9419-8f42dcacf3ae /tidb/cdc/default/default/upstream/7489433739627791549 {"id":7489433739627791549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50a74319-de1c-471c-9419-8f42dcacf3ae {"id":"50a74319-de1c-471c-9419-8f42dcacf3ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769688} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5ad48d8 50a74319-de1c-471c-9419-8f42dcacf3ae /tidb/cdc/default/default/upstream/7489433739627791549 {"id":7489433739627791549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50a74319-de1c-471c-9419-8f42dcacf3ae {"id":"50a74319-de1c-471c-9419-8f42dcacf3ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769688} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5ad48d8 50a74319-de1c-471c-9419-8f42dcacf3ae /tidb/cdc/default/default/upstream/7489433739627791549 {"id":7489433739627791549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7489433739627791549,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:28:10.923052861+08:00","start_ts":457118761008758785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118761008758785,"checkpoint_ts":457118761008758785,"checkpoint_time":"2025-04-04 20:28:07.686"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7489433739627791549,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:28:11.055618507+08:00","start_ts":457118761008758785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118761008758785,"checkpoint_ts":457118761008758785,"checkpoint_time":"2025-04-04 20:28:07.686"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7489433739627791549,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:28:11.221739846+08:00","start_ts":457118761008758785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118761008758785,"checkpoint_ts":457118761008758785,"checkpoint_time":"2025-04-04 20:28:07.686"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:28:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24720285-f76f-4ad7-8030-fcb113d439f5 {"id":"24720285-f76f-4ad7-8030-fcb113d439f5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769688} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5a7d9e5 24720285-f76f-4ad7-8030-fcb113d439f5 /tidb/cdc/default/default/upstream/7489433733051773213 {"id":7489433733051773213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24720285-f76f-4ad7-8030-fcb113d439f5 {"id":"24720285-f76f-4ad7-8030-fcb113d439f5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769688} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5a7d9e5 24720285-f76f-4ad7-8030-fcb113d439f5 /tidb/cdc/default/default/upstream/7489433733051773213 + grep -q 'failed to get info:' {"id":7489433733051773213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24720285-f76f-4ad7-8030-fcb113d439f5 {"id":"24720285-f76f-4ad7-8030-fcb113d439f5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769688} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5a7d9e5 24720285-f76f-4ad7-8030-fcb113d439f5 /tidb/cdc/default/default/upstream/7489433733051773213 {"id":7489433733051773213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118760708866049 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19684.out cli changefeed create --start-ts=457118760708866049 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489433733051773213,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:28:11.455990939+08:00","start_ts":457118760708866049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118760708866049,"checkpoint_ts":457118760708866049,"checkpoint_time":"2025-04-04 20:28:06.542"} PASS wait process cdc.test exit for 1-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [2025/04/04 20:28:13.027 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:28:13.028 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:28:13.028 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:28:13.029 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/04 20:28:13.032 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:28:13.032 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118754653339689] [resolvedTs=457118761207201795] [2025/04/04 20:28:13.033 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/04 20:28:13.033 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/04 20:28:13.033 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:28:13.033 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:28:13.033 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:28:13.036 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.421636ms] [2025/04/04 20:28:13.041 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:28:13.044 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:28:13.045 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:28:13.045 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:28:13.045 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:28:13.046 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:28:13.046 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:28:13.045 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:28:13.046 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:28:13.046 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:28:13.051 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:28:13.054 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:28:13.054 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:28:14.900 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.867704677s] [2025/04/04 20:28:14.901 +08:00] [ERROR] [redo.go:272] ["ignore unsupported DDL"] [ddl="{\"StartTs\":457118754653339684,\"CommitTs\":457118754653339689,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_s3\",\"Table\":\"t1\",\"TableID\":118,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/04 20:28:17.368 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118761207201795] [2025/04/04 20:28:17.368 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118761207201795] [2025/04/04 20:28:17.469 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118761207201795] [2025/04/04 20:28:17.469 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118761207201795] [2025/04/04 20:28:17.771 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457118761207201795] [2025/04/04 20:28:17.772 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457118761207201795] [2025/04/04 20:28:17.872 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=457118761207201795] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:28:17.872 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] Apply redo log successfully check diff successfully [Fri Apr 4 20:28:18 CST 2025] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433798454703623 TiDB Cluster ID. tikv_gc_leader_uuid 6580317eb60000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:30524, start at 2025-04-04 20:28:17.01234724 +0800 CST m=+2.430260186 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:17.057 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:17.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:17.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433798454703623 TiDB Cluster ID. tikv_gc_leader_uuid 6580317eb60000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:30524, start at 2025-04-04 20:28:17.01234724 +0800 CST m=+2.430260186 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:17.057 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:17.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:17.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433794409009346 TiDB Cluster ID. tikv_gc_leader_uuid 6580317ee14000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-0-p0352, pid:30596, start at 2025-04-04 20:28:17.174022997 +0800 CST m=+2.515250007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:17.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:17.176 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:17.176 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/vector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/vector/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/vector/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/vector/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/vector/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 17596 24.6 0.6 4864360 2504984 ? Sl 20:27 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 17597 24.3 0.6 4851044 2476952 ? Sl 20:27 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 17598 33.0 0.6 4886376 2541704 ? Sl 20:27 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 17602 31.7 0.6 4874084 2526180 ? Sl 20:27 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6360 0 --:--:-- --:--:-- --:--:-- 6394 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:28:19.892","puller_resolved_ts":"2025-04-04 20:28:12.893","last_synced_ts":"2025-04-04 20:28:13.393","now_ts":"2025-04-04 20:28:20.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:28:19.892","puller_resolved_ts":"2025-04-04' '20:28:12.893","last_synced_ts":"2025-04-04' '20:28:13.393","now_ts":"2025-04-04' '20:28:20.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:28:19.892","puller_resolved_ts":"2025-04-04' '20:28:12.893","last_synced_ts":"2025-04-04' '20:28:13.393","now_ts":"2025-04-04' '20:28:20.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.31847.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118764573392898 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118764573392898 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:28:22 CST 2025] <<<<<< START cdc server in vector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.3188031882.out server --log-file /tmp/tidb_cdc_test/vector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/vector/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:28:25 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e643e535-69a6-4ced-bf6c-595eb56400cf {"id":"e643e535-69a6-4ced-bf6c-595eb56400cf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769703} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5db91e5 e643e535-69a6-4ced-bf6c-595eb56400cf /tidb/cdc/default/default/upstream/7489433798454703623 {"id":7489433798454703623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e643e535-69a6-4ced-bf6c-595eb56400cf {"id":"e643e535-69a6-4ced-bf6c-595eb56400cf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769703} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5db91e5 e643e535-69a6-4ced-bf6c-595eb56400cf /tidb/cdc/default/default/upstream/7489433798454703623 {"id":7489433798454703623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e643e535-69a6-4ced-bf6c-595eb56400cf {"id":"e643e535-69a6-4ced-bf6c-595eb56400cf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769703} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c5db91e5 e643e535-69a6-4ced-bf6c-595eb56400cf /tidb/cdc/default/default/upstream/7489433798454703623 {"id":7489433798454703623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.31947.out cli changefeed create --start-ts=457118764573392898 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 3d5fa344-0ae1-4d5c-93d5-0ffe74dae08e Info: {"upstream_id":7489433798454703623,"namespace":"default","id":"3d5fa344-0ae1-4d5c-93d5-0ffe74dae08e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:28:26.194698694+08:00","start_ts":457118764573392898,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118764573392898,"checkpoint_ts":457118764573392898,"checkpoint_time":"2025-04-04 20:28:21.284"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table multi_tables_ddl_test.finish_mark exists check table exists success + set +x + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 457118765412515858, "checkpoint_time": "2025-04-04 20:28:24.485", "error": null }' + echo '{ "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 457118765412515858, "checkpoint_time": "2025-04-04 20:28:24.485", "error": null }' { "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 457118765412515858, "checkpoint_time": "2025-04-04 20:28:24.485", "error": null } ++ echo '{' '"upstream_id":' 7489433739627791549, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 457118765412515858, '"checkpoint_time":' '"2025-04-04' '20:28:24.485",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489433739627791549, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 457118765412515858, '"checkpoint_time":' '"2025-04-04' '20:28:24.485",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 457118766146781192, "checkpoint_time": "2025-04-04 20:28:27.286", "error": null }' + echo '{ "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 457118766146781192, "checkpoint_time": "2025-04-04 20:28:27.286", "error": null }' { "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 457118766146781192, "checkpoint_time": "2025-04-04 20:28:27.286", "error": null } ++ echo '{' '"upstream_id":' 7489433739627791549, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 457118766146781192, '"checkpoint_time":' '"2025-04-04' '20:28:27.286",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489433739627791549, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 457118766146781192, '"checkpoint_time":' '"2025-04-04' '20:28:27.286",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 457118765097943048, "checkpoint_time": "2025-04-04 20:28:23.285", "error": { "time": "2025-04-04T20:28:24.34038416+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 457118765097943048, "checkpoint_time": "2025-04-04 20:28:23.285", "error": { "time": "2025-04-04T20:28:24.34038416+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7489433739627791549, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 457118765097943048, "checkpoint_time": "2025-04-04 20:28:23.285", "error": { "time": "2025-04-04T20:28:24.34038416+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '138', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489433739627791549, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457118765097943048, '"checkpoint_time":' '"2025-04-04' '20:28:23.285",' '"error":' '{' '"time":' '"2025-04-04T20:28:24.34038416+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''138'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489433739627791549, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457118765097943048, '"checkpoint_time":' '"2025-04-04' '20:28:23.285",' '"error":' '{' '"time":' '"2025-04-04T20:28:24.34038416+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''138'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '138', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:28:30 CST 2025] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:28:37 CST 2025] <<<<<< run test case vector success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433893979762125 TiDB Cluster ID. tikv_gc_leader_uuid 65803195360000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:19066, start at 2025-04-04 20:28:40.057700014 +0800 CST m=+2.133006168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:40.065 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:40.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:40.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433893979762125 TiDB Cluster ID. tikv_gc_leader_uuid 65803195360000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:19066, start at 2025-04-04 20:28:40.057700014 +0800 CST m=+2.133006168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:40.065 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:40.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:40.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489433892697434577 TiDB Cluster ID. tikv_gc_leader_uuid 658031954d40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-s-qq2bm, pid:19141, start at 2025-04-04 20:28:40.150622551 +0800 CST m=+2.164716274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:30:40.158 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:28:40.117 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:18:40.117 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 4 20:28:44 CST 2025] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2041120413.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:28:47 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a64e9a46-46b7-4f72-8136-c4bc417d192b {"id":"a64e9a46-46b7-4f72-8136-c4bc417d192b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c634c9d5 a64e9a46-46b7-4f72-8136-c4bc417d192b /tidb/cdc/default/default/upstream/7489433893979762125 {"id":7489433893979762125,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a64e9a46-46b7-4f72-8136-c4bc417d192b {"id":"a64e9a46-46b7-4f72-8136-c4bc417d192b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c634c9d5 a64e9a46-46b7-4f72-8136-c4bc417d192b /tidb/cdc/default/default/upstream/7489433893979762125 {"id":7489433893979762125,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a64e9a46-46b7-4f72-8136-c4bc417d192b {"id":"a64e9a46-46b7-4f72-8136-c4bc417d192b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c634c9d5 a64e9a46-46b7-4f72-8136-c4bc417d192b /tidb/cdc/default/default/upstream/7489433893979762125 {"id":7489433893979762125,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table partition_table2.t2 not exists for 1-th check, retry later table partition_table2.t2 not exists for 2-th check, retry later table partition_table2.t2 not exists for 3-th check, retry later table partition_table2.t2 not exists for 4-th check, retry later table partition_table2.t2 not exists for 5-th check, retry later table partition_table2.t2 not exists for 6-th check, retry later table partition_table2.t2 not exists for 7-th check, retry later table partition_table2.t2 exists ____________________________________ *************************** 1. row *************************** primary_ts: 457118758993920000 secondary_ts: 457118759176110112 primary_ts is 457118758993920000 start_ts is 457118757487378433 check successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:29:02 CST 2025] <<<<<< run test case syncpoint_check_ts success! >>>>>> wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:29:11 CST 2025] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2059020592.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:29:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {UpstreamID:7489433893979762125 Namespace:default ID:093cc8c9-e59d-4304-9a78-922fdb1a1d82 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:28:47.513068102 +0800 CST StartTs:457118771425574917 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038835f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118771438682120} {CheckpointTs:457118777074778117 MinTableBarrierTs:457118778398867458 AdminJobType:noop} span: {tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73d98024-3c32-4d8a-aef6-f8b60dd0093a {"id":"73d98024-3c32-4d8a-aef6-f8b60dd0093a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769752} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c634cb2c 73d98024-3c32-4d8a-aef6-f8b60dd0093a /tidb/cdc/default/default/changefeed/info/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"upstream-id":7489433893979762125,"namespace":"default","changefeed-id":"093cc8c9-e59d-4304-9a78-922fdb1a1d82","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:28:47.513068102+08:00","start-ts":457118771425574917,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118771438682120} /tidb/cdc/default/default/changefeed/status/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"checkpoint-ts":457118777074778117,"min-table-barrier-ts":457118778398867458,"admin-job-type":0} /tidb/cdc/default/default/task/position/73d98024-3c32-4d8a-aef6-f8b60dd0093a/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433893979762125 {"id":7489433893979762125,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {UpstreamID:7489433893979762125 Namespace:default ID:093cc8c9-e59d-4304-9a78-922fdb1a1d82 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:28:47.513068102 +0800 CST StartTs:457118771425574917 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038835f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118771438682120} {CheckpointTs:457118777074778117 MinTableBarrierTs:457118778398867458 AdminJobType:noop} span: {tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73d98024-3c32-4d8a-aef6-f8b60dd0093a {"id":"73d98024-3c32-4d8a-aef6-f8b60dd0093a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769752} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c634cb2c 73d98024-3c32-4d8a-aef6-f8b60dd0093a /tidb/cdc/default/default/changefeed/info/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"upstream-id":7489433893979762125,"namespace":"default","changefeed-id":"093cc8c9-e59d-4304-9a78-922fdb1a1d82","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:28:47.513068102+08:00","start-ts":457118771425574917,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118771438682120} /tidb/cdc/default/default/changefeed/status/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"checkpoint-ts":457118777074778117,"min-table-barrier-ts":457118778398867458,"admin-job-type":0} /tidb/cdc/default/default/task/position/73d98024-3c32-4d8a-aef6-f8b60dd0093a/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433893979762125 {"id":7489433893979762125,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {UpstreamID:7489433893979762125 Namespace:default ID:093cc8c9-e59d-4304-9a78-922fdb1a1d82 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-04 20:28:47.513068102 +0800 CST StartTs:457118771425574917 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038835f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457118771438682120} {CheckpointTs:457118777074778117 MinTableBarrierTs:457118778398867458 AdminJobType:noop} span: {tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating span: {tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}, resolvedTs: 457118777074778117, checkpointTs: 457118777074778117, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73d98024-3c32-4d8a-aef6-f8b60dd0093a {"id":"73d98024-3c32-4d8a-aef6-f8b60dd0093a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769752} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c634cb2c 73d98024-3c32-4d8a-aef6-f8b60dd0093a /tidb/cdc/default/default/changefeed/info/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"upstream-id":7489433893979762125,"namespace":"default","changefeed-id":"093cc8c9-e59d-4304-9a78-922fdb1a1d82","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-04T20:28:47.513068102+08:00","start-ts":457118771425574917,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"igno+ grep -q 'etcd info' re-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457118771438682120} /tidb/cdc/default/default/changefeed/status/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"checkpoint-ts":457118777074778117,"min-table-barrier-ts":457118778398867458,"admin-job-type":0} /tidb/cdc/default/default/task/position/73d98024-3c32-4d8a-aef6-f8b60dd0093a/093cc8c9-e59d-4304-9a78-922fdb1a1d82 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489433893979762125 {"id":7489433893979762125,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434081494073219 TiDB Cluster ID. tikv_gc_leader_uuid 658031bf6b40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:15198, start at 2025-04-04 20:29:23.253187065 +0800 CST m=+2.206469367 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:31:23.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:29:23.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:19:23.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434081494073219 TiDB Cluster ID. tikv_gc_leader_uuid 658031bf6b40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:15198, start at 2025-04-04 20:29:23.253187065 +0800 CST m=+2.206469367 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:31:23.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:29:23.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:19:23.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434079312249707 TiDB Cluster ID. tikv_gc_leader_uuid 658031bf9d00003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:15275, start at 2025-04-04 20:29:23.447510763 +0800 CST m=+2.318725968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:31:23.455 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:29:23.444 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:19:23.444 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.16515.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118781982900225 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118781982900225 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 20:29:29 CST 2025] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1656516567.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:29:32 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/535f865c-0cf9-4368-b213-6b6c5b9ee67f {"id":"535f865c-0cf9-4368-b213-6b6c5b9ee67f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769769} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c6dd3bf4 535f865c-0cf9-4368-b213-6b6c5b9ee67f /tidb/cdc/default/default/upstream/7489434081494073219 {"id":7489434081494073219,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/535f865c-0cf9-4368-b213-6b6c5b9ee67f {"id":"535f865c-0cf9-4368-b213-6b6c5b9ee67f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769769} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c6dd3bf4 535f865c-0cf9-4368-b213-6b6c5b9ee67f /tidb/cdc/default/default/upstream/7489434081494073219 {"id":7489434081494073219,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/535f865c-0cf9-4368-b213-6b6c5b9ee67f {"id":"535f865c-0cf9-4368-b213-6b6c5b9ee67f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769769} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c6dd3bf4 535f865c-0cf9-4368-b213-6b6c5b9ee67f /tidb/cdc/default/default/upstream/7489434081494073219 {"id":7489434081494073219,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.16617.out cli changefeed create --start-ts=457118781982900225 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 5d23aaca-de29-4718-bf4b-f5dc56b06658 Info: {"upstream_id":7489434081494073219,"namespace":"default","id":"5d23aaca-de29-4718-bf4b-f5dc56b06658","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T20:29:32.757832601+08:00","start_ts":457118781982900225,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118781982900225,"checkpoint_ts":457118781982900225,"checkpoint_time":"2025-04-04 20:29:27.696"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:29:39 CST 2025] <<<<<< run test case server_config_compatibility success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434245483262406 TiDB Cluster ID. tikv_gc_leader_uuid 658031e3d74000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:17729, start at 2025-04-04 20:30:00.575589004 +0800 CST m=+2.234801542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:00.583 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:00.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:00.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434245483262406 TiDB Cluster ID. tikv_gc_leader_uuid 658031e3d74000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:17729, start at 2025-04-04 20:30:00.575589004 +0800 CST m=+2.234801542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:00.583 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:00.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:00.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434236376991393 TiDB Cluster ID. tikv_gc_leader_uuid 658031e3fac0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:17807, start at 2025-04-04 20:30:00.725753868 +0800 CST m=+2.293481532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:00.733 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:00.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:00.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.19100.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118791772667905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118791772667905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "dotransactions"="false" "mysql.user"="root" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.db"="changefeed_dup_error_restart" "updateproportion"="0" "operationcount"="0" "threadcount"="2" "workload"="core" "readproportion"="0" "scanproportion"="0" "recordcount"="10" "insertproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 8.668374ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1848.0, Avg(us): 1638, Min(us): 1053, Max(us): 3273, 95th(us): 4000, 99th(us): 4000 [Fri Apr 4 20:30:06 CST 2025] <<<<<< START cdc server in changefeed_dup_error_restart case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLDuplicateEntryError=5%return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.1915619158.out server --log-file /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:30:09 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a9c1bdd0-64f5-4e47-9090-adf89dd9fc80 {"id":"a9c1bdd0-64f5-4e47-9090-adf89dd9fc80","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c770e0f1 a9c1bdd0-64f5-4e47-9090-adf89dd9fc80 /tidb/cdc/default/default/upstream/7489434245483262406 {"id":7489434245483262406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a9c1bdd0-64f5-4e47-9090-adf89dd9fc80 {"id":"a9c1bdd0-64f5-4e47-9090-adf89dd9fc80","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c770e0f1 a9c1bdd0-64f5-4e47-9090-adf89dd9fc80 /tidb/cdc/default/default/upstream/7489434245483262406 {"id":7489434245483262406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a9c1bdd0-64f5-4e47-9090-adf89dd9fc80 {"id":"a9c1bdd0-64f5-4e47-9090-adf89dd9fc80","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769806} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c770e0f1 a9c1bdd0-64f5-4e47-9090-adf89dd9fc80 /tidb/cdc/default/default/upstream/7489434245483262406 {"id":7489434245483262406,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.19220.out cli changefeed create --start-ts=457118791772667905 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 9a0bf969-0c2e-4d87-a6f1-c72514aea101 Info: {"upstream_id":7489434245483262406,"namespace":"default","id":"9a0bf969-0c2e-4d87-a6f1-c72514aea101","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:30:10.013815922+08:00","start_ts":457118791772667905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118791772667905,"checkpoint_ts":457118791772667905,"checkpoint_time":"2025-04-04 20:30:05.041"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6565 0 --:--:-- --:--:-- --:--:-- 6696 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 20:30:19.465","puller_resolved_ts":"2025-04-04 20:30:12.416","last_synced_ts":"2025-04-04 20:28:03.365","now_ts":"2025-04-04 20:30:20.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:30:19.465","puller_resolved_ts":"2025-04-04' '20:30:12.416","last_synced_ts":"2025-04-04' '20:28:03.365","now_ts":"2025-04-04' '20:30:20.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:30:19.465","puller_resolved_ts":"2025-04-04' '20:30:12.416","last_synced_ts":"2025-04-04' '20:28:03.365","now_ts":"2025-04-04' '20:30:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 23489 0 --:--:-- --:--:-- --:--:-- 24100 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:28:19.892","puller_resolved_ts":"2025-04-04 20:28:19.892","last_synced_ts":"2025-04-04 20:28:13.393","now_ts":"2025-04-04 20:30:30.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:28:19.892","puller_resolved_ts":"2025-04-04' '20:28:19.892","last_synced_ts":"2025-04-04' '20:28:13.393","now_ts":"2025-04-04' '20:30:30.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:28:19.892","puller_resolved_ts":"2025-04-04' '20:28:19.892","last_synced_ts":"2025-04-04' '20:28:13.393","now_ts":"2025-04-04' '20:30:30.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434416212908064 TiDB Cluster ID. tikv_gc_leader_uuid 6580320ae340004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:18061, start at 2025-04-04 20:30:40.561690841 +0800 CST m=+2.194704150 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:40.569 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:40.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:40.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434416212908064 TiDB Cluster ID. tikv_gc_leader_uuid 6580320ae340004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:18061, start at 2025-04-04 20:30:40.561690841 +0800 CST m=+2.194704150 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:40.569 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:40.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:40.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434415829682858 TiDB Cluster ID. tikv_gc_leader_uuid 6580320af980007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-z-q4v78, pid:18140, start at 2025-04-04 20:30:40.658093716 +0800 CST m=+2.233011021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:40.666 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:40.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:40.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table changefeed_dup_error_restart.finish_mark_1 exists check diff successfully ***************** properties ***************** "workload"="core" "insertproportion"="0" "threadcount"="2" "mysql.port"="4000" "dotransactions"="false" "scanproportion"="0" "readallfields"="true" "operationcount"="0" "readproportion"="0" "recordcount"="10" "mysql.db"="changefeed_dup_error_restart" "mysql.user"="root" "updateproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 4.328408ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3095.7, Avg(us): 717, Min(us): 469, Max(us): 1606, 95th(us): 2000, 99th(us): 2000 + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19443.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118802267078657 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118802267078657 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118802267078657 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 20:30:46 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1947519477.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:30:49 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a73dae5-dd51-4d63-a0ec-38e70594d52d {"id":"3a73dae5-dd51-4d63-a0ec-38e70594d52d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769846} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c80d3fe5 3a73dae5-dd51-4d63-a0ec-38e70594d52d /tidb/cdc/default/default/upstream/7489434416212908064 {"id":7489434416212908064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a73dae5-dd51-4d63-a0ec-38e70594d52d {"id":"3a73dae5-dd51-4d63-a0ec-38e70594d52d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769846} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c80d3fe5 3a73dae5-dd51-4d63-a0ec-38e70594d52d /tidb/cdc/default/default/upstream/7489434416212908064 {"id":7489434416212908064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a73dae5-dd51-4d63-a0ec-38e70594d52d {"id":"3a73dae5-dd51-4d63-a0ec-38e70594d52d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769846} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c80d3fe5 3a73dae5-dd51-4d63-a0ec-38e70594d52d /tidb/cdc/default/default/upstream/7489434416212908064 {"id":7489434416212908064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118802267078657 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19532.out cli changefeed create --start-ts=457118802267078657 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489434416212908064,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:30:49.962088561+08:00","start_ts":457118802267078657,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118802267078657,"checkpoint_ts":457118802267078657,"checkpoint_time":"2025-04-04 20:30:45.074"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + sleep 20 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434457565020070 TiDB Cluster ID. tikv_gc_leader_uuid 6580321515c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:20895, start at 2025-04-04 20:30:50.982926721 +0800 CST m=+2.316819425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:50.991 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:51.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:51.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434457565020070 TiDB Cluster ID. tikv_gc_leader_uuid 6580321515c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:20895, start at 2025-04-04 20:30:50.982926721 +0800 CST m=+2.316819425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:50.991 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:51.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:51.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434455640765344 TiDB Cluster ID. tikv_gc_leader_uuid 658032150b4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:20975, start at 2025-04-04 20:30:50.951854298 +0800 CST m=+2.206872185 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:32:50.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:30:50.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:20:50.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22234.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118804952481793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118804952481793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118804952481793 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 20:30:56 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2226622268.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:30:59 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/166dda8b-93f6-46c9-beda-afcf00a775ff {"id":"166dda8b-93f6-46c9-beda-afcf00a775ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769857} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c83575e5 166dda8b-93f6-46c9-beda-afcf00a775ff /tidb/cdc/default/default/upstream/7489434457565020070 {"id":7489434457565020070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/166dda8b-93f6-46c9-beda-afcf00a775ff {"id":"166dda8b-93f6-46c9-beda-afcf00a775ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769857} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c83575e5 166dda8b-93f6-46c9-beda-afcf00a775ff /tidb/cdc/default/default/upstream/7489434457565020070 {"id":7489434457565020070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/166dda8b-93f6-46c9-beda-afcf00a775ff {"id":"166dda8b-93f6-46c9-beda-afcf00a775ff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769857} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c83575e5 166dda8b-93f6-46c9-beda-afcf00a775ff /tidb/cdc/default/default/upstream/7489434457565020070 {"id":7489434457565020070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118804952481793 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22322.out cli changefeed create --start-ts=457118804952481793 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489434457565020070,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:31:00.230957635+08:00","start_ts":457118804952481793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118804952481793,"checkpoint_ts":457118804952481793,"checkpoint_time":"2025-04-04 20:30:55.318"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 20891 4.6 0.0 2839888 185496 ? Sl 20:30 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743769848655824283.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 20895 10.4 0.0 2364520 214180 ? Sl 20:30 0:01 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743769848659026435.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 20975 10.6 0.0 2546656 217812 ? Sl 20:30 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743769848737038534.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 5517 0 --:--:-- --:--:-- --:--:-- 5651 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:31:06.718","puller_resolved_ts":"2025-04-04 20:31:01.668","last_synced_ts":"2025-04-04 20:31:02.168","now_ts":"2025-04-04 20:31:07.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:31:06.718","puller_resolved_ts":"2025-04-04' '20:31:01.668","last_synced_ts":"2025-04-04' '20:31:02.168","now_ts":"2025-04-04' '20:31:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:31:06.718","puller_resolved_ts":"2025-04-04' '20:31:01.668","last_synced_ts":"2025-04-04' '20:31:02.168","now_ts":"2025-04-04' '20:31:07.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 114k 0 --:--:-- --:--:-- --:--:-- 117k + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:30:45.074","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 20:31:11.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:30:45.074","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:31:11.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:30:45.074","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:31:11.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster table changefeed_dup_error_restart.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:31:13 CST 2025] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/partition_table/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["partition_table.?*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2025/04/04 20:31:19.074 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/04 20:31:19.075 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/04 20:31:19.075 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/04 20:31:19.075 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/04 20:31:19.075 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457118777074778117] [resolvedTs=457118810144243718] [2025/04/04 20:31:19.076 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:31:19.076 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:31:19.076 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:31:19.076 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.247391ms] [2025/04/04 20:31:19.077 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.580791ms] [2025/04/04 20:31:19.081 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:31:19.084 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/04 20:31:19.085 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/04 20:31:19.085 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/04 20:31:19.085 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/04 20:31:19.085 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/04 20:31:19.089 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/04 20:31:19.091 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/04 20:31:19.091 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/04 20:31:19.091 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118778608582705,\"CommitTs\":457118778621427713,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:31:21.100 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/04 20:31:21.100 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118778621427713] [DDL="alter table t1 drop partition p1"] [2025/04/04 20:31:21.155 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=55.095324ms] [sql="alter table t1 drop partition p1"] [2025/04/04 20:31:21.155 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118778739392538,\"CommitTs\":457118778739392544,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:31:21.356 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118778739392544] [DDL="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [2025/04/04 20:31:21.394 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=37.566083ms] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [2025/04/04 20:31:21.394 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118778870464540,\"CommitTs\":457118778883571716,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:31:21.794 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118778883571716] [DDL="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [2025/04/04 20:31:21.829 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=34.192694ms] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [2025/04/04 20:31:21.829 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457118779224358915,\"CommitTs\":457118779224358919,\"Query\":\"ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":64,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/04 20:31:22.229 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457118779224358919] [DDL="ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))"] + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Fri Apr 4 20:31:22 CST 2025] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Fri Apr 4 20:31:22 CST 2025] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/safe_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/04 20:31:23.331 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=1.102159246s] [sql="ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))"] [2025/04/04 20:31:23.332 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.332 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.432 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.432 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.532 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.532 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.632 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.632 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.732 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.732 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.833 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.833 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.933 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:23.933 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.033 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.033 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.133 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.133 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.233 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.233 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.333 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.334 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.434 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.434 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}"] [checkpointTs=457118810144243718] [2025/04/04 20:31:24.534 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=47] [appliedDDLCount=4] [currentCheckpoint=457118810144243718] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/04 20:31:24.534 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] Apply redo log successfully check diff successfully [Fri Apr 4 20:31:24 CST 2025] <<<<<< run test case consistent_partition_table success! >>>>>> Database INFORMATION_SCHEMA METRICS_SCHEMA PERFORMANCE_SCHEMA mysql partition_table partition_table2 sys test \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/safe_mode Starting Upstream PD... [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:31:34 CST 2025] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434638657713162 TiDB Cluster ID. tikv_gc_leader_uuid 6580323f8580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:20452, start at 2025-04-04 20:31:34.459018447 +0800 CST m=+2.249032117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:33:34.467 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:31:34.471 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:21:34.471 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434638657713162 TiDB Cluster ID. tikv_gc_leader_uuid 6580323f8580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:20452, start at 2025-04-04 20:31:34.459018447 +0800 CST m=+2.249032117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:33:34.467 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:31:34.471 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:21:34.471 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489434647385229252 TiDB Cluster ID. tikv_gc_leader_uuid 6580323fab80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-j-f0stg, pid:20529, start at 2025-04-04 20:31:34.585625691 +0800 CST m=+2.316706870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:33:34.629 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:31:34.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:21:34.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/safe_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/safe_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:31:37 CST 2025] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> [Fri Apr 4 20:31:38 CST 2025] <<<<<< START cdc server in safe_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.safe_mode.2178921791.out server --log-file /tmp/tidb_cdc_test/safe_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/safe_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:31:40 CST 2025] <<<<<< run test case multi_topics_v2 success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:31:41 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a769a220-990e-4369-962d-9afea76537aa {"id":"a769a220-990e-4369-962d-9afea76537aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769899} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c8dd72ea a769a220-990e-4369-962d-9afea76537aa /tidb/cdc/default/default/upstream/7489434638657713162 {"id":7489434638657713162,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a769a220-990e-4369-962d-9afea76537aa {"id":"a769a220-990e-4369-962d-9afea76537aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769899} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c8dd72ea a769a220-990e-4369-962d-9afea76537aa /tidb/cdc/default/default/upstream/7489434638657713162 {"id":7489434638657713162,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a769a220-990e-4369-962d-9afea76537aa {"id":"a769a220-990e-4369-962d-9afea76537aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743769899} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600c8dd72ea a769a220-990e-4369-962d-9afea76537aa /tidb/cdc/default/default/upstream/7489434638657713162 {"id":7489434638657713162,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.safe_mode.cli.21848.out cli changefeed create '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?safe-mode=true' Create changefeed successfully! ID: 3cfe4cc8-6938-4fbf-b827-c5c52d5b376e Info: {"upstream_id":7489434638657713162,"namespace":"default","id":"3cfe4cc8-6938-4fbf-b827-c5c52d5b376e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-04T20:31:42.18730009+08:00","start_ts":457118817221607426,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118817221607426,"checkpoint_ts":457118817221607426,"checkpoint_time":"2025-04-04 20:31:42.121"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=mysql + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri Apr 4 20:31:43 CST 2025] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri Apr 4 20:31:43 CST 2025] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:31:47 CST 2025] <<<<<< run test case csv_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:31:50 CST 2025] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:31:53 CST 2025] <<<<<< run test case csv_storage_partition_table success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table safe_mode.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 20:32:25 CST 2025] <<<<<< run test case safe_mode success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:34 CST 2025] <<<<<< run test case kafka_big_messages success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:37 CST 2025] <<<<<< run test case kafka_compression success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:41 CST 2025] <<<<<< run test case kafka_messages success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:44 CST 2025] <<<<<< run test case kafka_sink_error_resume success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:47 CST 2025] <<<<<< run test case mq_sink_lost_callback success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:50 CST 2025] <<<<<< run test case mq_sink_dispatcher success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:53 CST 2025] <<<<<< run test case kafka_column_selector success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:32:57 CST 2025] <<<<<< run test case kafka_column_selector_avro success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: mysql... <<================= =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:33:00 CST 2025] <<<<<< run test case lossy_ddl success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:33:03 CST 2025] <<<<<< run test case storage_csv_update success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_update_pk_clustered/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:33:06 CST 2025] <<<<<< run test case csv_storage_update_pk_clustered success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_update_pk_nonclustered/run.sh using Sink-Type: mysql... <<================= [Fri Apr 4 20:33:09 CST 2025] <<<<<< run test case csv_storage_update_pk_nonclustered success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6779 0 --:--:-- --:--:-- --:--:-- 6906 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 20:33:16.918","puller_resolved_ts":"2025-04-04 20:33:10.918","last_synced_ts":"2025-04-04 20:31:02.168","now_ts":"2025-04-04 20:33:17.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:33:16.918","puller_resolved_ts":"2025-04-04' '20:33:10.918","last_synced_ts":"2025-04-04' '20:31:02.168","now_ts":"2025-04-04' '20:33:17.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '20:33:16.918","puller_resolved_ts":"2025-04-04' '20:33:10.918","last_synced_ts":"2025-04-04' '20:31:02.168","now_ts":"2025-04-04' '20:33:17.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489435166669641899 TiDB Cluster ID. tikv_gc_leader_uuid 658032b60500002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:23450, start at 2025-04-04 20:33:35.765220891 +0800 CST m=+2.209123703 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:35:35.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:33:35.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:23:35.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489435166669641899 TiDB Cluster ID. tikv_gc_leader_uuid 658032b60500002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:23450, start at 2025-04-04 20:33:35.765220891 +0800 CST m=+2.209123703 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:35:35.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:33:35.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:23:35.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489435171553547359 TiDB Cluster ID. tikv_gc_leader_uuid 658032b61e0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-231-1-w0717, pid:23526, start at 2025-04-04 20:33:35.879161071 +0800 CST m=+2.251132868 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-20:35:35.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-20:33:35.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-20:23:35.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24829.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457118848179240961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457118848179240961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457118848179240961 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 20:33:41 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2486924871.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 12:33:44 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ff6e219-4b7a-407e-bc65-f035f6b7c52d {"id":"0ff6e219-4b7a-407e-bc65-f035f6b7c52d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743770022} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600cabf72e4 0ff6e219-4b7a-407e-bc65-f035f6b7c52d /tidb/cdc/default/default/upstream/7489435166669641899 {"id":7489435166669641899,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ff6e219-4b7a-407e-bc65-f035f6b7c52d {"id":"0ff6e219-4b7a-407e-bc65-f035f6b7c52d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743770022} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600cabf72e4 0ff6e219-4b7a-407e-bc65-f035f6b7c52d /tidb/cdc/default/default/upstream/7489435166669641899 {"id":7489435166669641899,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0ff6e219-4b7a-407e-bc65-f035f6b7c52d {"id":"0ff6e219-4b7a-407e-bc65-f035f6b7c52d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743770022} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319600cabf72e4 0ff6e219-4b7a-407e-bc65-f035f6b7c52d /tidb/cdc/default/default/upstream/7489435166669641899 {"id":7489435166669641899,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457118848179240961 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24921.out cli changefeed create --start-ts=457118848179240961 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489435166669641899,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T20:33:45.171524304+08:00","start_ts":457118848179240961,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457118848179240961,"checkpoint_ts":457118848179240961,"checkpoint_time":"2025-04-04 20:33:40.215"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 91727 0 --:--:-- --:--:-- --:--:-- 100k + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 20:33:40.215","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 20:34:06.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:33:40.215","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:34:06.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '20:33:40.215","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '20:34:06.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Fri Apr 4 20:34:17 CST 2025] <<<<<< run test case synced_status success! >>>>>>' [Fri Apr 4 20:34:17 CST 2025] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-231/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Declarative: Post Actions) [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $WEBHOOK_URL [Pipeline] { [Pipeline] sh Warning: A secret was passed to "sh" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [WEBHOOK_URL] See https://jenkins.io/redirect/groovy-string-interpolation for details. + curl -X POST **** -H 'Content-Type: application/json' -d '{ "msg_type": "text", "content": { "text": "CDC MySQL Integration Test: SUCCESS\nBranch: master\nBuild URL: https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/231/\nJob Page: https://prow.tidb.net/?repo=pingcap%2Ftiflow&type=periodic\n" } }' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 499 100 77 100 422 88 484 --:--:-- --:--:-- --:--:-- 483 100 499 100 77 100 422 88 484 --:--:-- --:--:-- --:--:-- 483 {"StatusCode":0,"StatusMessage":"success","code":0,"data":{},"msg":"success"} [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS